diff --git a/.github/workflows/Documentation.yml b/.github/workflows/Documentation.yml new file mode 100644 index 000000000..7017e7e46 --- /dev/null +++ b/.github/workflows/Documentation.yml @@ -0,0 +1,47 @@ +name: Documentation + +on: + push: + branches: + - 'master' + tags: '*' + workflow_dispatch: + schedule: + # Run on the 23rd hour every day + - cron: '0 23 * * *' + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Julia + uses: julia-actions/setup-julia@v1 + with: + version: '1' + - name: Set up Ruby 2.6 + uses: actions/setup-ruby@v1 + with: + ruby-version: '2.6.x' + - name: Install dependencies + run: | + julia --project=docs -e ' + using Pkg + Pkg.activate(".") + Pkg.develop("Turing") + Pkg.instantiate() + Pkg.update()' + - name: Build and deploy (master) + run: | + julia --project=docs --color=yes make.jl + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} + TURING_VERSION: dev + - name: Build and deploy (stable) + run: | + git -C ~/.julia/dev/Turing checkout $(git -C ~/.julia/dev/Turing tag --sort version:refname | tail -n 1) + julia --project=docs --color=yes make.jl $(git -C ~/.julia/dev/Turing tag --sort version:refname | tail -n 1) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DOCUMENTER_KEY: ${{ secrets.DOCUMENTER_KEY }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 73c5104a6..648aebff7 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ _site/ .sass-cache/ .jekyll-cache/ .jekyll-metadata -.gems/ \ No newline at end of file +.gems/ +Manifest.toml \ No newline at end of file diff --git a/Manifest.toml b/Manifest.toml deleted file mode 100644 index 3876b682e..000000000 --- a/Manifest.toml +++ /dev/null @@ -1,648 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -[[AbstractFFTs]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "051c95d6836228d120f5f4b984dd5aba1624f716" -uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" -version = "0.5.0" - -[[AbstractMCMC]] -deps = ["ProgressMeter", "Random", "StatsBase"] -git-tree-sha1 = "b2c89b3c9c63dbcc0842293c713143a9bbdff79f" -uuid = "80f14c24-f653-4e6a-9b94-39d6b0f70001" -version = "0.1.0" - -[[Adapt]] -deps = ["LinearAlgebra"] -git-tree-sha1 = "c88cfc7f9c1f9f8633cddf0b56e86302b70f64c5" -uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" -version = "1.0.1" - -[[AdvancedHMC]] -deps = ["ArgCheck", "InplaceOps", "LazyArrays", "LinearAlgebra", "Parameters", "ProgressMeter", "Random", "Statistics", "StatsBase", "StatsFuns"] -git-tree-sha1 = "e3ef41a8746e0f1c0b8eba98257f3d0d00e5922b" -uuid = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" -version = "0.2.14" - -[[ArgCheck]] -git-tree-sha1 = "e14de95bcfacd85e00f5369958a38bb72827bce2" -uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" -version = "1.1.0" - -[[Arpack]] -deps = ["Arpack_jll", "Libdl", "LinearAlgebra"] -git-tree-sha1 = "2ff92b71ba1747c5fdd541f8fc87736d82f40ec9" -uuid = "7d9fca2a-8960-54d3-9f78-7d1dccf2cb97" -version = "0.4.0" - -[[Arpack_jll]] -deps = ["Libdl", "OpenBLAS_jll", "Pkg"] -git-tree-sha1 = "68a90a692ddc0eb72d69a6993ca26e2a923bf195" -uuid = "68821587-b530-5797-8361-c406ea357684" -version = "3.5.0+2" - -[[ArrayInterface]] -deps = ["LinearAlgebra", "Requires", "SparseArrays"] -git-tree-sha1 = "8683deff18ed6cd123e2b4d4147d2dc21f19391b" -uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "2.6.1" - -[[ArrayLayouts]] -deps = ["FillArrays", "LinearAlgebra"] -git-tree-sha1 = "bc779df8d73be70e4e05a63727d3a4dfb4c52b1f" -uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" -version = "0.1.5" - -[[AxisAlgorithms]] -deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"] -git-tree-sha1 = "a4d07a1c313392a77042855df46c5f534076fab9" -uuid = "13072b0f-2c55-5437-9ae7-d433b7a33950" -version = "1.0.0" - -[[AxisArrays]] -deps = ["Dates", "IntervalSets", "IterTools", "RangeArrays"] -git-tree-sha1 = "d63ba0315a1d287c9467e61e932578f2fdd048e0" -uuid = "39de3d68-74b9-583c-8d2d-e117c070f3a9" -version = "0.3.3" - -[[Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[BenchmarkTools]] -deps = ["JSON", "Logging", "Printf", "Statistics", "UUIDs"] -git-tree-sha1 = "9e62e66db34540a0c919d72172cc2f642ac71260" -uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -version = "0.5.0" - -[[Bijectors]] -deps = ["Distributions", "ForwardDiff", "LinearAlgebra", "MappedArrays", "Random", "Reexport", "Requires", "Roots", "StatsFuns", "Tracker"] -git-tree-sha1 = "0b7b2f292e770eb32e5e1966cc4b5c605a154022" -uuid = "76274a88-744f-5084-9051-94815aaf08c4" -version = "0.4.0" - -[[BinDeps]] -deps = ["Libdl", "Pkg", "SHA", "URIParser", "Unicode"] -git-tree-sha1 = "66158ad56b4bf6cc8413b37d0b7bc52402682764" -uuid = "9e28174c-4ba2-5203-b857-d8d62c4213ee" -version = "1.0.0" - -[[BinaryProvider]] -deps = ["Libdl", "SHA"] -git-tree-sha1 = "5b08ed6036d9d3f0ee6369410b830f8873d4024c" -uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" -version = "0.5.8" - -[[CategoricalArrays]] -deps = ["Compat", "DataAPI", "Future", "JSON", "Missings", "Printf", "Reexport", "Statistics", "Unicode"] -git-tree-sha1 = "23d7324164c89638c18f6d7f90d972fa9c4fa9fb" -uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597" -version = "0.7.7" - -[[Combinatorics]] -deps = ["Polynomials"] -git-tree-sha1 = "140cc833258df8e5aafabdb068875ac0c256be23" -uuid = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" -version = "1.0.0" - -[[CommonSubexpressions]] -deps = ["Test"] -git-tree-sha1 = "efdaf19ab11c7889334ca247ff4c9f7c322817b0" -uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" -version = "0.2.0" - -[[Compat]] -deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] -git-tree-sha1 = "0198d18b28c093bef39872a22f1a897218a925f5" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "3.8.0" - -[[DataAPI]] -git-tree-sha1 = "674b67f344687a88310213ddfa8a2b3c76cc4252" -uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.1.0" - -[[DataFrames]] -deps = ["CategoricalArrays", "Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "Missings", "PooledArrays", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "7d5bf815cc0b30253e3486e8ce2b93bf9d0faff6" -uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "0.20.2" - -[[DataStructures]] -deps = ["InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "5a431d46abf2ef2a4d5d00bd0ae61f651cf854c8" -uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.17.10" - -[[DataValueInterfaces]] -git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" -uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" -version = "1.0.0" - -[[Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[DelimitedFiles]] -deps = ["Mmap"] -uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" - -[[DiffResults]] -deps = ["StaticArrays"] -git-tree-sha1 = "da24935df8e0c6cf28de340b958f6aac88eaa0cc" -uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" -version = "1.0.2" - -[[DiffRules]] -deps = ["NaNMath", "Random", "SpecialFunctions"] -git-tree-sha1 = "eb0c34204c8410888844ada5359ac8b96292cfd1" -uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" -version = "1.0.1" - -[[Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - -[[Distributions]] -deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"] -git-tree-sha1 = "e063d0b5d27180b98edacd2b1cb90ecfbc171385" -uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" -version = "0.21.12" - -[[DistributionsAD]] -deps = ["Combinatorics", "Distributions", "FiniteDifferences", "ForwardDiff", "LinearAlgebra", "PDMats", "Random", "StatsFuns", "Test", "Tracker", "Zygote"] -git-tree-sha1 = "3dd6ce2ad107e7362e08a6834d3bbdd3e4f07b7b" -uuid = "ced4e74d-a319-5a8a-b0ac-84af2272839c" -version = "0.1.1" - -[[DocStringExtensions]] -deps = ["LibGit2", "Markdown", "Pkg", "Test"] -git-tree-sha1 = "88bb0edb352b16608036faadcc071adda068582a" -uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.8.1" - -[[Documenter]] -deps = ["Base64", "Dates", "DocStringExtensions", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] -git-tree-sha1 = "0be9bf63e854a2408c2ecd3c600d68d4d87d8a73" -uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" -version = "0.24.2" - -[[DocumenterMarkdown]] -deps = ["Documenter", "Test"] -git-tree-sha1 = "c302ba512683c3db462ee4eff718ae6fedcbf380" -uuid = "997ab1e6-3595-5248-9280-8efb232c3433" -version = "0.2.0" - -[[EllipsisNotation]] -git-tree-sha1 = "65dad386e877850e6fce4fc77f60fe75a468ce9d" -uuid = "da5c29d0-fa7d-589e-88eb-ea29b0a81949" -version = "0.4.0" - -[[FFTW]] -deps = ["AbstractFFTs", "FFTW_jll", "IntelOpenMP_jll", "Libdl", "LinearAlgebra", "MKL_jll", "Reexport"] -git-tree-sha1 = "109d82fa4b00429f9afcce873e9f746f11f018d3" -uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.2.0" - -[[FFTW_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "ddb57f4cf125243b4aa4908c94d73a805f3cbf2c" -uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" -version = "3.3.9+4" - -[[FillArrays]] -deps = ["LinearAlgebra", "Random", "SparseArrays"] -git-tree-sha1 = "85c6b57e2680fa28d5c8adc798967377646fbf66" -uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" -version = "0.8.5" - -[[FiniteDiff]] -deps = ["ArrayInterface", "LinearAlgebra", "Requires", "SparseArrays", "StaticArrays"] -git-tree-sha1 = "f60e5d6944975f7140bde67278e10b6b01fb4f29" -uuid = "6a86dc24-6348-571c-b903-95158fe2bd41" -version = "2.3.0" - -[[FiniteDifferences]] -deps = ["LinearAlgebra", "Printf"] -git-tree-sha1 = "f7f69a1cb10c04958b3da91913ce5af992924150" -uuid = "26cc04aa-876d-5657-8c51-4c34ba976000" -version = "0.9.2" - -[[ForwardDiff]] -deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"] -git-tree-sha1 = "88b082d492be6b63f967b6c96b352e25ced1a34c" -uuid = "f6369f11-7733-5829-9624-2563aa707210" -version = "0.10.9" - -[[Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" - -[[IRTools]] -deps = ["InteractiveUtils", "MacroTools", "Test"] -git-tree-sha1 = "1a4355e4b5b50be2311ebb644f34f3306dbd0410" -uuid = "7869d1d1-7146-5819-86e3-90919afe41df" -version = "0.3.1" - -[[InplaceOps]] -deps = ["LinearAlgebra", "Test"] -git-tree-sha1 = "50b41d59e7164ab6fda65e71049fee9d890731ff" -uuid = "505f98c9-085e-5b2c-8e89-488be7bf1f34" -version = "0.3.0" - -[[IntelOpenMP_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "fb8e1c7a5594ba56f9011310790e03b5384998d6" -uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" -version = "2018.0.3+0" - -[[InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[Interpolations]] -deps = ["AxisAlgorithms", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"] -git-tree-sha1 = "f5bf159a7705e2a705b0effa1be0c3d18e288fe1" -uuid = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" -version = "0.12.5" - -[[IntervalSets]] -deps = ["Dates", "EllipsisNotation", "Statistics"] -git-tree-sha1 = "8a83c4788fdc2e32d08d90214834ec2dc1a54f57" -uuid = "8197267c-284f-5f27-9208-e0e47529a953" -version = "0.4.0" - -[[InvertedIndices]] -deps = ["Test"] -git-tree-sha1 = "15732c475062348b0165684ffe28e85ea8396afc" -uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" -version = "1.0.0" - -[[IterTools]] -git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18" -uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" -version = "1.3.0" - -[[IteratorInterfaceExtensions]] -git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" -uuid = "82899510-4779-5014-852e-03e436cf321d" -version = "1.0.0" - -[[JSON]] -deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.0" - -[[KernelDensity]] -deps = ["Distributions", "FFTW", "Interpolations", "Optim", "StatsBase", "Test"] -git-tree-sha1 = "c1048817fe5711f699abc8fabd47b1ac6ba4db04" -uuid = "5ab0869b-81aa-558d-bb23-cbf5423bbe9b" -version = "0.5.1" - -[[LazyArrays]] -deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra", "MacroTools", "StaticArrays"] -git-tree-sha1 = "d10de258d0c999350d433e4eed35201c9d790bcd" -uuid = "5078a376-72f3-5289-bfd5-ec5146d43c02" -version = "0.14.11" - -[[LibGit2]] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[Libtask]] -deps = ["BinaryProvider", "Libdl", "Pkg"] -git-tree-sha1 = "4f39d56bf87bf17f5a04370bd036baf0d1fe8cc9" -uuid = "6f1fad26-d15e-5dc8-ae53-837a1d7b8c9f" -version = "0.3.1" - -[[LineSearches]] -deps = ["LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "Printf", "Test"] -git-tree-sha1 = "54eb90e8dbe745d617c78dee1d6ae95c7f6f5779" -uuid = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" -version = "7.0.1" - -[[LinearAlgebra]] -deps = ["Libdl"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[LogDensityProblems]] -deps = ["ArgCheck", "BenchmarkTools", "DiffResults", "DocStringExtensions", "Parameters", "Random", "Requires", "TransformVariables"] -git-tree-sha1 = "10058bfce11b71237e3ea9975b4153f8207d536e" -uuid = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c" -version = "0.9.1" - -[[Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[MCMCChains]] -deps = ["AbstractMCMC", "AxisArrays", "DataFrames", "Distributions", "KernelDensity", "LinearAlgebra", "Random", "RecipesBase", "Serialization", "Showoff", "SpecialFunctions", "Statistics", "StatsBase"] -git-tree-sha1 = "5774b1d0266fc300bfb9bd0f8149ad3fdee0c99c" -uuid = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" -version = "0.4.1" - -[[MKL_jll]] -deps = ["IntelOpenMP_jll", "Libdl", "Pkg"] -git-tree-sha1 = "720629cc8cbd12c146ca01b661fd1a6cf66e2ff4" -uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" -version = "2019.0.117+2" - -[[MacroTools]] -deps = ["DataStructures", "Markdown", "Random"] -git-tree-sha1 = "07ee65e03e28ca88bc9a338a3726ae0c3efaa94b" -uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.4" - -[[MappedArrays]] -git-tree-sha1 = "e2a02fe7ee86a10c707ff1756ab1650b40b140bb" -uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" -version = "0.2.2" - -[[Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[Missings]] -deps = ["DataAPI"] -git-tree-sha1 = "de0a5ce9e5289f27df672ffabef4d1e5861247d5" -uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "0.4.3" - -[[Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[NLSolversBase]] -deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"] -git-tree-sha1 = "7c4e66c47848562003250f28b579c584e55becc0" -uuid = "d41bc354-129a-5804-8e4c-c37616107c6c" -version = "7.6.1" - -[[NNlib]] -deps = ["BinaryProvider", "Libdl", "LinearAlgebra", "Requires", "Statistics"] -git-tree-sha1 = "d9f196d911f55aeaff11b11f681b135980783824" -uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" -version = "0.6.6" - -[[NaNMath]] -git-tree-sha1 = "928b8ca9b2791081dc71a51c55347c27c618760f" -uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" -version = "0.3.3" - -[[OffsetArrays]] -git-tree-sha1 = "6a35d9446b40ae5004cd7bd0f1ae3505528c7fd6" -uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" -version = "1.0.3" - -[[OpenBLAS_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "139adbff69e8149e68824994b68f06a61a5a2797" -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.7+8" - -[[Optim]] -deps = ["Compat", "FillArrays", "LineSearches", "LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "PositiveFactorizations", "Printf", "SparseArrays", "StatsBase"] -git-tree-sha1 = "1c44e1ac9c39892ff74de99ec9f7428f03660851" -uuid = "429524aa-4258-5aef-a3af-852621145aeb" -version = "0.20.4" - -[[OrderedCollections]] -deps = ["Random", "Serialization", "Test"] -git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.1.0" - -[[PDMats]] -deps = ["Arpack", "LinearAlgebra", "SparseArrays", "SuiteSparse", "Test"] -git-tree-sha1 = "2fc6f50ddd959e462f0a2dbc802ddf2a539c6e35" -uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" -version = "0.9.12" - -[[Parameters]] -deps = ["OrderedCollections"] -git-tree-sha1 = "b62b2558efb1eef1fa44e4be5ff58a515c287e38" -uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" -version = "0.12.0" - -[[Parsers]] -deps = ["Dates", "Test"] -git-tree-sha1 = "0c16b3179190d3046c073440d94172cfc3bb0553" -uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "0.3.12" - -[[Pkg]] -deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Test", "UUIDs"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" - -[[Polynomials]] -deps = ["LinearAlgebra", "RecipesBase"] -git-tree-sha1 = "1185511cac8ab9d0b658b663eae34fe9a95d4332" -uuid = "f27b6e38-b328-58d1-80ce-0feddd5e7a45" -version = "0.6.1" - -[[PooledArrays]] -deps = ["DataAPI"] -git-tree-sha1 = "b1333d4eced1826e15adbdf01a4ecaccca9d353c" -uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" -version = "0.5.3" - -[[PositiveFactorizations]] -deps = ["LinearAlgebra", "Test"] -git-tree-sha1 = "127c47b91990c101ee3752291c4f45640eeb03d1" -uuid = "85a6dd25-e78a-55b7-8502-1745935b8125" -version = "0.2.3" - -[[Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[ProgressMeter]] -deps = ["Distributed", "Printf"] -git-tree-sha1 = "ea1f4fa0ff5e8b771bf130d87af5b7ef400760bd" -uuid = "92933f4c-e287-5a05-a399-4b506db050ca" -version = "1.2.0" - -[[QuadGK]] -deps = ["DataStructures", "LinearAlgebra"] -git-tree-sha1 = "dc84e810393cfc6294248c9032a9cdacc14a3db4" -uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" -version = "2.3.1" - -[[REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[Random]] -deps = ["Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[RangeArrays]] -git-tree-sha1 = "b9039e93773ddcfc828f12aadf7115b4b4d225f5" -uuid = "b3c3ace0-ae52-54e7-9d0b-2c1406fd6b9d" -version = "0.3.2" - -[[Ratios]] -git-tree-sha1 = "37d210f612d70f3f7d57d488cb3b6eff56ad4e41" -uuid = "c84ed2f1-dad5-54f0-aa8e-dbefe2724439" -version = "0.4.0" - -[[RecipesBase]] -git-tree-sha1 = "7bdce29bc9b2f5660a6e5e64d64d91ec941f6aa2" -uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" -version = "0.7.0" - -[[Reexport]] -deps = ["Pkg"] -git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0" -uuid = "189a3867-3050-52da-a836-e630ba90ab69" -version = "0.2.0" - -[[Requires]] -deps = ["Test"] -git-tree-sha1 = "f6fbf4ba64d295e146e49e021207993b6b48c7d1" -uuid = "ae029012-a4dd-5104-9daa-d747884805df" -version = "0.5.2" - -[[Rmath]] -deps = ["Random", "Rmath_jll"] -git-tree-sha1 = "86c5647b565873641538d8f812c04e4c9dbeb370" -uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" -version = "0.6.1" - -[[Rmath_jll]] -deps = ["Libdl", "Pkg"] -git-tree-sha1 = "1660f8fefbf5ab9c67560513131d4e933012fc4b" -uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" -version = "0.2.2+0" - -[[Roots]] -deps = ["Printf"] -git-tree-sha1 = "869a9990ec6347862d59040d00416ecd0683b965" -uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665" -version = "1.0.0" - -[[SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" - -[[Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[SharedArrays]] -deps = ["Distributed", "Mmap", "Random", "Serialization"] -uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" - -[[Showoff]] -deps = ["Dates"] -git-tree-sha1 = "e032c9df551fb23c9f98ae1064de074111b7bc39" -uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" -version = "0.3.1" - -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[SortingAlgorithms]] -deps = ["DataStructures", "Random", "Test"] -git-tree-sha1 = "03f5898c9959f8115e30bc7226ada7d0df554ddd" -uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "0.3.1" - -[[SparseArrays]] -deps = ["LinearAlgebra", "Random"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[SpecialFunctions]] -deps = ["BinDeps", "BinaryProvider", "Libdl"] -git-tree-sha1 = "3bdd374b6fd78faf0119b8c5d538788dbf910c6e" -uuid = "276daf66-3868-5448-9aa4-cd146d93841b" -version = "0.8.0" - -[[StaticArrays]] -deps = ["LinearAlgebra", "Random", "Statistics"] -git-tree-sha1 = "5a3bcb6233adabde68ebc97be66e95dcb787424c" -uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "0.12.1" - -[[Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" - -[[StatsBase]] -deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"] -git-tree-sha1 = "19bfcb46245f69ff4013b3df3b977a289852c3a1" -uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" -version = "0.32.2" - -[[StatsFuns]] -deps = ["Rmath", "SpecialFunctions"] -git-tree-sha1 = "f290ddd5fdedeadd10e961eb3f4d3340f09d030a" -uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" -version = "0.9.4" - -[[SuiteSparse]] -deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] -uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" - -[[TableTraits]] -deps = ["IteratorInterfaceExtensions"] -git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e" -uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" -version = "1.0.0" - -[[Tables]] -deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"] -git-tree-sha1 = "242b7fde70b8bc6a30d6476adf17ca3cf1ced6ee" -uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.0.3" - -[[Test]] -deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[Tracker]] -deps = ["Adapt", "DiffRules", "ForwardDiff", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Printf", "Random", "Requires", "SpecialFunctions", "Statistics", "Test"] -git-tree-sha1 = "86929a5811dca5ce76c65a1d3fecda92d90c2e49" -uuid = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" -version = "0.2.6" - -[[TransformVariables]] -deps = ["ArgCheck", "DocStringExtensions", "ForwardDiff", "LinearAlgebra", "MacroTools", "Parameters", "Pkg", "Random"] -git-tree-sha1 = "dd4c257cb7a997bcabb5c5afbe62e544c1a750ce" -uuid = "84d833dd-6860-57f9-a1a7-6da5db126cff" -version = "0.3.9" - -[[Turing]] -deps = ["AbstractMCMC", "AdvancedHMC", "Bijectors", "BinaryProvider", "Distributions", "DistributionsAD", "ForwardDiff", "Libtask", "LinearAlgebra", "LogDensityProblems", "MCMCChains", "MacroTools", "Markdown", "PDMats", "ProgressMeter", "Random", "Reexport", "Requires", "SpecialFunctions", "Statistics", "StatsFuns", "Tracker", "Zygote"] -git-tree-sha1 = "b2da884516f2e3a14b0878a543cb40ab29b55bf5" -uuid = "fce5fe82-541a-59a6-adf8-730c64b5f9a0" -version = "0.7.4" - -[[URIParser]] -deps = ["Test", "Unicode"] -git-tree-sha1 = "6ddf8244220dfda2f17539fa8c9de20d6c575b69" -uuid = "30578b45-9adc-5946-b283-645ec420af67" -version = "0.4.0" - -[[UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[WoodburyMatrices]] -deps = ["LinearAlgebra", "SparseArrays"] -git-tree-sha1 = "68f000f67654d07318d734b364a31233e465f49a" -uuid = "efce3f68-66dc-5838-9240-27a6d6f5f9b6" -version = "0.5.1" - -[[Zygote]] -deps = ["ArrayLayouts", "DiffRules", "FFTW", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "4a3c8decdf1d498cd13fe29827350d727a3b3854" -uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" -version = "0.4.10" - -[[ZygoteRules]] -deps = ["MacroTools"] -git-tree-sha1 = "b3b4882cc9accf6731a08cc39543fbc6b669dca8" -uuid = "700de1a5-db45-46bc-99cf-38207098b444" -version = "0.2.0" diff --git a/Project.toml b/Project.toml index 11eefb5d9..ed4edec1e 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,11 @@ +name = "docs" + [deps] AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" +AdvancedMH = "5b7e9947-ddc0-4b3f-9b55-0d8042f74170" Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DocumenterMarkdown = "997ab1e6-3595-5248-9280-8efb232c3433" +DocumenterTools = "35a29f4d-8980-5a13-9543-d66fff28ecb8" Libtask = "6f1fad26-d15e-5dc8-ae53-837a1d7b8c9f" Turing = "fce5fe82-541a-59a6-adf8-730c64b5f9a0" diff --git a/_data/toc.yml b/_data/toc.yml deleted file mode 100644 index c4051ff86..000000000 --- a/_data/toc.yml +++ /dev/null @@ -1,99 +0,0 @@ -- title: "USING TURING" - url: "docs/using-turing/get-started" - children: - - title: "Getting Started" - url: "docs/using-turing/get-started" - - - title: "Quick Start" - url: "docs/using-turing/quick-start" - - - title: "Guide" - url: "docs/using-turing/guide" - - - title: "Advanced Usage" - url: "docs/using-turing/advanced" - - - title: "Automatic Differentiation" - url: "docs/using-turing/autodiff" - - - title: "Performance Tips" - url: "docs/using-turing/performancetips" - - - title: "Using DynamicHMC" - url: "docs/using-turing/dynamichmc" - - - title: "Sampler Visualization" - url: "docs/using-turing/sampler-viz" - -- title: "FOR DEVELOPERS" - url: "docs/for-developers/" - children: - - title: "Turing Compiler Design" - url: "docs/for-developers/compiler" - - - title: "Interface Guide" - url: "docs/for-developers/interface" - - - title: "How Turing implements AbstractMCMC" - url: "docs/for-developers/how_turing_implements_abstractmcmc" - - - title: "Variational Inference" - url: "docs/for-developers/variational_inference" - -- title: "TUTORIALS" - url: "tutorials" - children: - - title: "Home" - url: "tutorials" - - - title: "Introduction to Turing" - url: "tutorials/0-introduction" - - - title: "Gaussian Mixture Models" - url: "tutorials/1-gaussianmixturemodel" - - - title: "Bayesian Logistic Regression" - url: "tutorials/2-logisticregression" - - - title: "Bayesian Neural Networks" - url: "tutorials/3-bayesnn" - - - title: "Hidden Markov Models" - url: "tutorials/4-bayeshmm" - - - title: "Linear Regression" - url: "tutorials/5-linearregression" - - - title: "Infinite Mixture Models" - url: "tutorials/6-infinitemixturemodel" - - - title: "Bayesian Poisson Regression" - url: "tutorials/7-poissonregression" - - - title: "Multinomial Logistic Regression" - url: "tutorials/8-multinomiallogisticregression" - - - title: "Variational Inference" - url: "tutorials/9-variationalinference" - - - title: "Bayesian Differential Equation" - url: "tutorials/10-bayesiandiffeq" - -- title: "API" - url: "docs/library" - children: - - title: "Turing" - url: "docs/library" - - title: "AdvancedHMC" - url: "docs/library/advancedhmc/" - - title: "Bijectors" - url: "docs/library/bijectors/" - -- title: "CONTRIBUTING" - url: "docs/contributing/guide" - children: - - title: "How to Contribute" - url: "docs/contributing/guide" - - - title: "Style Guide" - url: "docs/contributing/style-guide" diff --git a/_docs/assets/Documenter.css b/_docs/assets/Documenter.css deleted file mode 100644 index d9af5d6c9..000000000 --- a/_docs/assets/Documenter.css +++ /dev/null @@ -1,18 +0,0 @@ -div.wy-menu-vertical ul.current li.toctree-l3 a { - font-weight: bold; -} - -a.documenter-source { - float: right; -} - -.documenter-methodtable pre { - margin-left: 0; - margin-right: 0; - margin-top: 0; - padding: 0; -} - -.documenter-methodtable pre.documenter-inline { - display: inline; -} diff --git a/_docs/assets/mathjaxhelper.js b/_docs/assets/mathjaxhelper.js deleted file mode 100644 index 3561b109f..000000000 --- a/_docs/assets/mathjaxhelper.js +++ /dev/null @@ -1,25 +0,0 @@ -MathJax.Hub.Config({ - "tex2jax": { - inlineMath: [['$','$'], ['\\(','\\)']], - processEscapes: true - } -}); -MathJax.Hub.Config({ - config: ["MMLorHTML.js"], - jax: [ - "input/TeX", - "output/HTML-CSS", - "output/NativeMML" - ], - extensions: [ - "MathMenu.js", - "MathZoom.js", - "TeX/AMSmath.js", - "TeX/AMSsymbols.js", - "TeX/autobold.js", - "TeX/autoload-all.js" - ] -}); -MathJax.Hub.Config({ - TeX: { equationNumbers: { autoNumber: "AMS" } } -}); diff --git a/_docs/contributing/guide.md b/_docs/contributing/guide.md deleted file mode 100644 index beaf3d1b6..000000000 --- a/_docs/contributing/guide.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: Contributing ---- - - - - - -# Contributing - - -Turing is an open source project. If you feel that you have some relevant skills and are interested in contributing, then please do get in touch. You can contribute by opening issues on GitHub or implementing things yourself and making a pull request. We would also appreciate example models written using Turing. - - -Turing has a [style guide]({{site.baseurl}}/docs/contributing/style-guide). It is not strictly necessary to review it before making a pull request, but you may be asked to change portions of your code to conform with the style guide before it is merged. - - - - - - -## How to Contribute - - - - - - -### Getting Started - - - * [Fork this repository](https://github.com/TuringLang/Turing.jl#fork-destination-box). - * Clone your fork on your local machine: `git clone https://github.com/your_username/Turing.jl`. - * Add a remote corresponding to this repository: - - -`git remote add upstream https://github.com/TuringLang/Turing.jl`. - - - - - - -### What Can I Do? - - -Look at the [issues](https://github.com/TuringLang/Turing.jl/issues) page to find an outstanding issue. For instance, you could implement new features, fix bugs or write example models. - - - - - - -### Git Workflow - - -For more information on how the Git workflow typically functions, please see the [GitHub's introduction](https://guides.github.com/introduction/flow/) or [Julia's contribution guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md). - diff --git a/_docs/contributing/style-guide.md b/_docs/contributing/style-guide.md deleted file mode 100644 index 3abdee766..000000000 --- a/_docs/contributing/style-guide.md +++ /dev/null @@ -1,735 +0,0 @@ ---- -title: Style Guide ---- - - - - - -# Style Guide - - -This style guide is adapted from [Invenia](https://invenia.ca/labs/)'s style guide. We would like to thank them for allowing us to access and use it. Please don't let not having read it stop you from contributing to Turing! No one will be annoyed if you open a PR whose style doesn't follow these conventions; we will just help you correct it before it gets merged. - - -These conventions were originally written at Invenia, taking inspiration from a variety of sources including Python's [PEP8](http://legacy.python.org/dev/peps/pep-0008), Julia's [Notes for Contributors](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md), and Julia's [Style Guide](https://docs.julialang.org/en/latest/manual/style-guide/). - - -What follows is a mixture of a verbatim copy of Invenia's original guide and some of our own modifications. - - - - - - -## A Word on Consistency - - -When adhering to this style it's important to realize that these are guidelines and not rules. This is [stated best in the PEP8](http://legacy.python.org/dev/peps/pep-0008/#a-foolish-consistency-is-the-hobgoblin-of-little-minds): - - -> A style guide is about consistency. Consistency with this style guide is important. Consistency within a project is more important. Consistency within one module or function is most important. - - - -> But most importantly: know when to be inconsistent – sometimes the style guide just doesn't apply. When in doubt, use your best judgment. Look at other examples and decide what looks best. And don't hesitate to ask! - - - - - - - -## Synopsis - - -Attempt to follow both the [Julia Contribution Guidelines](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#general-formatting-guidelines-for-julia-code-contributions), the [Julia Style Guide](https://docs.julialang.org/en/latest/manual/style-guide/), and this guide. When convention guidelines conflict this guide takes precedence (known conflicts will be noted in this guide). - - - * Use 4 spaces per indentation level, no tabs. - * Try to adhere to a 92 character line length limit. - * Use upper camel case convention for [modules](https://docs.julialang.org/en/latest/manual/modules/) and [types](https://docs.julialang.org/en/latest/manual/types/). - * Use lower case with underscores for method names (note: Julia code likes to use lower case without underscores). - * Comments are good, try to explain the intentions of the code. - * Use whitespace to make the code more readable. - * No whitespace at the end of a line (trailing whitespace). - * Avoid padding brackets with spaces. ex. `Int64(value)` preferred over `Int64( value )`. - - - - - - -## Editor Configuration - - - - - - -### Sublime Text Settings - - -If you are a user of Sublime Text we recommend that you have the following options in your Julia syntax specific settings. To modify these settings first open any Julia file (`*.jl`) in Sublime Text. Then navigate to: `Preferences > Settings - More > Syntax Specific - User` - - -```json -{ - "translate_tabs_to_spaces": true, - "tab_size": 4, - "trim_trailing_white_space_on_save": true, - "ensure_newline_at_eof_on_save": true, - "rulers": [92] -} -``` - - - - - - -### Vim Settings - - -If you are a user of Vim we recommend that you add the following options to your `.vimrc` file. - - -``` -set tabstop=4 " Sets tabstops to a width of four columns. -set softtabstop=4 " Determines the behaviour of TAB and BACKSPACE keys with expandtab. -set shiftwidth=4 " Determines the results of >>, <<, and ==. - -au FileType julia setlocal expandtab " Replaces tabs with spaces. -au FileType julia setlocal colorcolumn=93 " Highlights column 93 to help maintain the 92 character line limit. -``` - - -By default, Vim seems to guess that `.jl` files are written in Lisp. To ensure that Vim recognizes Julia files you can manually have it check for the `.jl` extension, but a better solution is to install [Julia-Vim](https://github.com/JuliaLang/julia-vim), which also includes proper syntax highlighting and a few cool other features. - - - - - - -### Atom Settings - - -Atom defaults preferred line length to 80 characters. We want that at 92 for julia. To change it: - - -1. Go to `Atom -> Preferences -> Packages`. -2. Search for the "language-julia" package and open the settings for it. -3. Find preferred line length (under "Julia Grammar") and change it to 92. - - - - - - -## Code Formatting - - - - - - -### Function Naming - - -Names of functions should describe an action or property irrespective of the type of the argument; the argument's type provides this information instead. For example, `buyfood(food)` should be `buy(food::Food)`. - - -Names of functions should usually be limited to one or two lowercase words. Ideally write `buyfood` not `buy_food`, but if you are writing a function whose name is hard to read without underscores then please do use them. - - - - - - -### Method Definitions - - -Only use short-form function definitions when they fit on a single line: - - -```julia -# Yes: -foo(x::Int64) = abs(x) + 3 -# No: -foobar(array_data::AbstractArray{T}, item::T) where {T<:Int64} = T[ - abs(x) * abs(item) + 3 for x in array_data -] - -# No: -foobar( - array_data::AbstractArray{T}, - item::T, -) where {T<:Int64} = T[abs(x) * abs(item) + 3 for x in array_data] -# Yes: -function foobar(array_data::AbstractArray{T}, item::T) where T<:Int64 - return T[abs(x) * abs(item) + 3 for x in array_data] -end -``` - - -When using long-form functions [always use the `return` keyword](https://groups.google.com/forum/#!topic/julia-users/4RVR8qQDrUg): - - -```julia -# Yes: -function fnc(x) - result = zero(x) - result += fna(x) - return result -end -# No: -function fnc(x) - result = zero(x) - result += fna(x) -end - -# Yes: -function Foo(x, y) - return new(x, y) -end -# No: -function Foo(x, y) - new(x, y) -end -``` - - -Functions definitions with parameter lines which exceed 92 characters should separate each parameter by a newline and indent by one-level: - - -```julia -# Yes: -function foobar( - df::DataFrame, - id::Symbol, - variable::Symbol, - value::AbstractString, - prefix::AbstractString="", -) - # code -end - -# Ok: -function foobar(df::DataFrame, id::Symbol, variable::Symbol, value::AbstractString, prefix::AbstractString="") - # code -end -# No: -function foobar(df::DataFrame, id::Symbol, variable::Symbol, value::AbstractString, - prefix::AbstractString="") - - # code -end -# No: -function foobar( - df::DataFrame, - id::Symbol, - variable::Symbol, - value::AbstractString, - prefix::AbstractString="", - ) - # code -end -``` - - - - - - -### Keyword Arguments - - -When calling a function always separate your keyword arguments from your positional arguments with a semicolon. This avoids mistakes in ambiguous cases (such as splatting a `Dict`). - - -```julia -# Yes: -xy = foo(x; y=3) -# No: -xy = foo(x, y=3) -``` - - - - - - -### Whitespace - - -Avoid extraneous whitespace in the following situations: - - - * Immediately inside parentheses, square brackets or braces. - - -```julia -Yes: spam(ham[1], [eggs]) -No: spam( ham[ 1 ], [ eggs ] ) -``` - - - * Immediately before a comma or semicolon: - - -```julia -Yes: if x == 4 @show(x, y); x, y = y, x end -No: if x == 4 @show(x , y) ; x , y = y , x end -``` - - - * When using ranges unless additional operators are used: - - -```julia -Yes: ham[1:9], ham[1:3:9], ham[1:3:end] -No: ham[1: 9], ham[1 : 3: 9] -``` - - -```julia -Yes: ham[lower:upper], ham[lower:step:upper] -Yes: ham[lower + offset : upper + offset] -Yes: ham[(lower + offset):(upper + offset)] -No: ham[lower + offset:upper + offset] -``` - - - * More than one space around an assignment (or other) operator to align it with another: - - -```julia -# Yes: -x = 1 -y = 2 -long_variable = 3 - -# No: -x = 1 -y = 2 -long_variable = 3 -``` - - - * When using parametric types: - - -```julia -# Yes: -f(a::AbstractArray{T, N}) where {T<:Real, N} = ... -g(a::AbstractArray{<:Real, N}) where {N} = ... - -# No: -f(a::AbstractArray{T, N}) where {T <: Real, N} = ... -g(a::AbstractArray{<: Real, N}) where {N} = ... -``` - - - * Always surround these binary operators with a single space on either side: assignment ($=$), [updating operators](https://docs.julialang.org/en/latest/manual/mathematical-operations/#Updating-operators-1) ($+=$, $-=$, etc.), [numeric comparisons operators](https://docs.julialang.org/en/latest/manual/mathematical-operations/#Numeric-Comparisons-1) ($==$, $<$, $>$, $!=$, etc.). Note that this guideline does not apply when performing assignment in method definitions. - - -```julia -Yes: i = i + 1 -No: i=i+1 - -Yes: submitted += 1 -No: submitted +=1 - -Yes: x^2 < y -No: x^2 - - - -### Comments - - -Comments should be used to state the intended behaviour of code. This is especially important when the code is doing something clever that may not be obvious upon first inspection. Avoid writing comments that state exactly what the code obviously does. - - -```julia -# Yes: -x = x + 1 # Compensate for border - -# No: -x = x + 1 # Increment x -``` - - -Comments that contradict the code are much worse than no comments. Always make a priority of keeping the comments up-to-date with code changes! - - -Comments should be complete sentences. If a comment is a phrase or sentence, its first word should be capitalized, unless it is an identifier that begins with a lower case letter (never alter the case of identifiers!). - - -If a comment is short, the period at the end can be omitted. Block comments generally consist of one or more paragraphs built out of complete sentences, and each sentence should end in a period. - - -Comments should be separated by at least two spaces from the expression and have a single space after the `#`. - - -When referencing Julia in documentation note that "Julia" refers to the programming language while "julia" (typically in backticks, e.g. `julia`) refers to the executable. - - -```julia - -# A commment -code - -# Another comment -more code - -TODO -``` - - - - - - -### Documentation - - -It is recommended that most modules, types and functions should have [docstrings](http://docs.julialang.org/en/latest/manual/documentation/). That being said, only exported functions are required to be documented. Avoid documenting methods like `==` as the built in docstring for the function already covers the details well. Try to document a function and not individual methods where possible as typically all methods will have similar docstrings. If you are adding a method to a function which was defined in `Base` or another package only add a docstring if the behaviour of your function deviates from the existing docstring. - - -Docstrings are written in [Markdown](https://en.wikipedia.org/wiki/Markdown) and should be concise. Docstring lines should be wrapped at 92 characters. - - -```julia -""" - bar(x[, y]) - -Compute the Bar index between `x` and `y`. If `y` is missing, compute the Bar index between -all pairs of columns of `x`. -""" -function bar(x, y) ... -``` - - -When types or methods have lots of parameters it may not be feasible to write a concise docstring. In these cases it is recommended you use the templates below. Note if a section doesn't apply or is overly verbose (for example "Throws" if your function doesn't throw an exception) it can be excluded. It is recommended that you have a blank line between the headings and the content when the content is of sufficient length. Try to be consistent within a docstring whether you use this additional whitespace. Note that the additional space is only for reading raw markdown and does not effect the rendered version. - - -Type Template (should be skipped if is redundant with the constructor(s) docstring): - - -```julia -""" - MyArray{T,N} - -My super awesome array wrapper! - -# Fields -- `data::AbstractArray{T,N}`: stores the array being wrapped -- `metadata::Dict`: stores metadata about the array -""" -struct MyArray{T,N} <: AbstractArray{T,N} - data::AbstractArray{T,N} - metadata::Dict -end -``` - - -Function Template (only required for exported functions): - - -```julia -""" - mysearch(array::MyArray{T}, val::T; verbose=true) where {T} -> Int - -Searches the `array` for the `val`. For some reason we don't want to use Julia's -builtin search :) - -# Arguments -- `array::MyArray{T}`: the array to search -- `val::T`: the value to search for - -# Keywords -- `verbose::Bool=true`: print out progress details - -# Returns -- `Int`: the index where `val` is located in the `array` - -# Throws -- `NotFoundError`: I guess we could throw an error if `val` isn't found. -""" -function mysearch(array::AbstractArray{T}, val::T) where T - ... -end -``` - - -If your method contains lots of arguments or keywords you may want to exclude them from the method signature on the first line and instead use `args...` and/or `kwargs...`. - - -```julia -""" - Manager(args...; kwargs...) -> Manager - -A cluster manager which spawns workers. - -# Arguments - -- `min_workers::Integer`: The minimum number of workers to spawn or an exception is thrown -- `max_workers::Integer`: The requested number of worker to spawn - -# Keywords - -- `definition::AbstractString`: Name of the job definition to use. Defaults to the - definition used within the current instance. -- `name::AbstractString`: ... -- `queue::AbstractString`: ... -""" -function Manager(...) - ... -end -``` - - -Feel free to document multiple methods for a function within the same docstring. Be careful to only do this for functions you have defined. - - -```julia -""" - Manager(max_workers; kwargs...) - Manager(min_workers:max_workers; kwargs...) - Manager(min_workers, max_workers; kwargs...) - -A cluster manager which spawns workers. - -# Arguments - -- `min_workers::Int`: The minimum number of workers to spawn or an exception is thrown -- `max_workers::Int`: The number of requested workers to spawn - -# Keywords - -- `definition::AbstractString`: Name of the job definition to use. Defaults to the - definition used within the current instance. -- `name::AbstractString`: ... -- `queue::AbstractString`: ... -""" -function Manager end - -``` - - -If the documentation for bullet-point exceeds 92 characters the line should be wrapped and slightly indented. Avoid aligning the text to the `:`. - - -```julia -""" -... - -# Keywords -- `definition::AbstractString`: Name of the job definition to use. Defaults to the - definition used within the current instance. -""" -``` - - -For additional details on documenting in Julia see the [official documentation](http://docs.julialang.org/en/latest/manual/documentation/). - - - - - - -## Test Formatting - - - - - - -### Testsets - - -Julia provides [test sets](https://docs.julialang.org/en/latest/stdlib/Test/#Working-with-Test-Sets-1) which allows developers to group tests into logical groupings. Test sets can be nested and ideally packages should only have a single "root" test set. It is recommended that the "runtests.jl" file contains the root test set which contains the remainder of the tests: - - -```julia -@testset "PkgExtreme" begin - include("arithmetic.jl") - include("utils.jl") -end -``` - - -The file structure of the `test` folder should mirror that of the `src` folder. Every file in `src` should have a complementary file in the `test` folder, containing tests relevant to that file's contents. - - - - - - -### Comparisons - - -Most tests are written in the form `@test x == y`. Since the `==` function doesn't take types into account tests like the following are valid: `@test 1.0 == 1`. Avoid adding visual noise into test comparisons: - - -```julia -# Yes: -@test value == 0 - -# No: -@test value == 0.0 -``` - - -In cases where you are checking the numerical validity of a model's parameter estimates, please use the `check_numerical` function found in `test/test_utils/numerical_tests.jl`. This function will evaluate a model's parameter estimates using tolerance levels `atol` and `rtol`. Testing will only be performed if you are running the test suite locally or if Travis is executing the "Numerical" testing stage. - - -Here is an example of usage: - - -```julia -# Check that m and s are plus or minus one from 1.5 and 2.2, respectively. -check_numerical(chain, [:m, :s], [1.5, 2.2], atol = 1.0) - -# Checks the estimates for a default gdemo model using values 1.5 and 2.0. -check_gdemo(chain, atol = 0.1) - -# Checks the estimates for a default MoG model. -check_MoGtest_default(chain, atol = 0.1) -``` - diff --git a/_docs/for-developers/compiler.md b/_docs/for-developers/compiler.md deleted file mode 100644 index d370a321e..000000000 --- a/_docs/for-developers/compiler.md +++ /dev/null @@ -1,320 +0,0 @@ ---- -title: Turing Compiler Design ---- - -In this section, the current design of Turing's model "compiler" is described which enables Turing to perform various types of Bayesian inference without changing the model definition. The "compiler" is essentially just a macro that rewrites the user's model definition to a function that generates a `Model` struct that Julia's dispatch can operate on and that Julia's compiler can successfully do type inference on for efficient machine code generation. - - - - - - -# Overview - - -The following terminology will be used in this section: - - - * `D`: observed data variables conditioned upon in the posterior, - * `P`: parameter variables distributed according to the prior distributions, these will also be referred to as random variables, - * `Model`: a fully defined probabilistic model with input data - - -`Turing`'s `@model` macro rewrites the user-provided function definition such that it can be used to instantiate a `Model` by passing in the observed data `D`. - - -The following are the main jobs of the `@model` macro: - - -1. Parse `~` and `.~` lines, e.g. `y .~ Normal.(c*x, 1.0)` -2. Figure out if a variable belongs to the data `D` and or to the parameters `P` -3. Enable the handling of missing data variables in `D` when defining a `Model` and treating them as parameter variables in `P` instead -4. Enable the tracking of random variables using the data structures `VarName` and `VarInfo` -5. Change `~`/`.~` lines with a variable in `P` on the LHS to a call to `tilde_assume` or `dot_tilde_assume` -6. Change `~`/`.~` lines with a variable in `D` on the LHS to a call to `tilde_observe` or `dot_tilde_observe` -7. Enable type stable automatic differentiation of the model using type parameters - - - - - - -## The model - - -A `model::Model` is a callable struct that one can sample from by calling - - -```julia -(model::Model)([rng, varinfo, sampler, context]) -``` - - -where `rng` is a random number generator (default: `Random.GLOBAL_RNG`), `varinfo` is a data structure that stores information about the random variables (default: `DynamicPPL.VarInfo()`), `sampler` is a sampling algorithm (default: `DynamicPPL.SampleFromPrior()`), and `context` is a sampling context that can, e.g., modify how the log probability is accumulated (default: `DynamicPPL.DefaultContext()`). - - -Sampling resets the log joint probability of `varinfo` and increases the evaluation counter of `sampler`. If `context` is a `LikelihoodContext`, only the log likelihood will be accumulated. With the `DefaultContext` the log joint probability of `P` and `D` is accumulated. - - -The `Model` struct contains the three internal fields `f`, `args` and `defaults`. When `model::Model` is called, then the internal function `model.f` is called as `model.f(rng, varinfo, sampler, context, model.args...)` (for multithreaded sampling, instead of `varinfo` a threadsafe wrapper is passed to `model.f`). The positional and keyword arguments that were passed to the user-defined model function when the model was created are saved as a `NamedTuple` in `model.args`. The default values of the positional and keyword arguments of the user-defined model functions, if any, are saved as a `NamedTuple` in `model.defaults`. They are used for constructing model instances with different arguments by the `logprob` and `prob` string macros. - - - - - - -# Example - - -Let's take the following model as an example: - - -```julia -@model function gauss(x = missing, y = 1.0, ::Type{TV} = Vector{Float64}) where {TV<:AbstractVector} - if x === missing - x = TV(undef, 3) - end - p = TV(undef, 2) - p[1] ~ InverseGamma(2, 3) - p[2] ~ Normal(0, 1.0) - @. x[1:2] ~ Normal(p[2], sqrt(p[1])) - x[3] ~ Normal() - y ~ Normal(p[2], sqrt(p[1])) -end -``` - - -The above call of the `@model` macro defines the function `gauss` with positional arguments `x`, `y`, and `::Type{TV}`, rewritten in such a way that every call of it returns a `model::Model`. Note that only the function body is modified by the `@model` macro, and the function signature is left untouched. It is also possible to implement models with keyword arguments such as - - -```julia -@model function gauss(::Type{TV} = Vector{Float64}; x = missing, y = 1.0) where {TV<:AbstractVector} - ... -end -``` - - -This would allow us to generate a model by calling `gauss(; x = rand(3))`. - - -If an argument has a default value `missing`, it is treated as a random variable. For variables which require an intialization because we need to loop or broadcast over its elements, such as `x` above, the following needs to be done: - - -```julia -if x === missing - x = ... -end -``` - - -Note that since `gauss` behaves like a regular function it is possible to define additional dispatches in a second step as well. For instance, we could achieve the same behaviour by - - -```julia -@model function gauss(x, y = 1.0, ::Type{TV} = Vector{Float64}) where {TV<:AbstractVector} - p = TV(undef, 2) - ... -end - -function gauss(::Missing, y = 1.0, ::Type{TV} = Vector{Float64}) where {TV<:AbstractVector} - return gauss(TV(undef, 3), y, TV) -end -``` - - -If `x` is sampled as a whole from a distribution and not indexed, e.g., `x ~ Normal(...)` or `x ~ MvNormal(...)`, there is no need to initialize it in an `if`-block. - - - - - - -## Step 1: Break up the model definition - - -First, the `@model` macro breaks up the user-provided function definition using `DynamicPPL.build_model_info`. This function returns a dictionary consisting of: - - - * `allargs_exprs`: The expressions of the positional and keyword arguments, without default values. - * `allargs_syms`: The names of the positional and keyword arguments, e.g., `[:x, :y, :TV]` above. - * `allargs_namedtuple`: An expression that constructs a `NamedTuple` of the positional and keyword arguments, e.g., `:((x = x, y = y, TV = TV))` above. - * `defaults_namedtuple`: An expression that constructs a `NamedTuple` of the default positional and keyword arguments, if any, e.g., `:((x = missing, y = 1, TV = Vector{Float64}))` above. - * `modeldef`: A dictionary with the name, arguments, and function body of the model definition, as returned by `MacroTools.splitdef`. - - - - - - -## Step 2: Generate the body of the internal model function - - -In a second step, `DynamicPPL.generate_mainbody` generates the main part of the transformed function body using the user-provided function body and the provided function arguments, without default values, for figuring out if a variable denotes an observation or a random variable. Hereby the function `DynamicPPL.generate_tilde` replaces the `L ~ R` lines in the model and the function `DynamicPPL.generate_dot_tilde` replaces the `@. L ~ R` and `L .~ R` lines in the model. - - -In the above example, `p[1] ~ InverseGamma(2, 3)` is replaced with something similar to - - -```julia -#= REPL[25]:6 =# -begin - var"##tmpright#323" = InverseGamma(2, 3) - var"##tmpright#323" isa Union{Distribution, AbstractVector{<:Distribution}} || throw(ArgumentError("Right-hand side of a ~ must be subtype of Distribution or a vector of Distributions.")) - var"##vn#325" = (DynamicPPL.VarName)(:p, ((1,),)) - var"##inds#326" = ((1,),) - p[1] = (DynamicPPL.tilde_assume)(_rng, _context, _sampler, var"##tmpright#323", var"##vn#325", var"##inds#326", _varinfo) -end -``` - - -Here the first line is a so-called line number node that enables more helpful error messages by providing users with the exact location of the error in their model definition. Then the right hand side (RHS) of the `~` is assigned to a variable (with an automatically generated name). We check that the RHS is a distribution or an array of distributions, otherwise an error is thrown. Next we extract a compact representation of the variable with its name and index (or indices). Finally, the `~` expression is replaced with a call to `DynamicPPL.tilde_assume` since the compiler figured out that `p[1]` is a random variable using the following heuristic: - - -1. If the symbol on the LHS of `~`, `:p` in this case, is not among the arguments to the model, `(:x, :y, :T)` in this case, it is a random variable. -2. If the symbol on the LHS of `~`, `:p` in this case, is among the arguments to the model but has a value of `missing`, it is a random variable. -3. If the value of the LHS of `~`, `p[1]` in this case, is `missing`, then it is a random variable. -4. Otherwise, it is treated as an observation. - - -The `DynamicPPL.tilde_assume` function takes care of sampling the random variable, if needed, and updating its value and the accumulated log joint probability in the `_varinfo` object. If `L ~ R` is an observation, `DynamicPPL.tilde_observe` is called with the same arguments except the random number generator `_rng` (since observations are never sampled). - - -A similar transformation is performed for expressions of the form `@. L ~ R` and `L .~ R`. For instance, `@. x[1:2] ~ Normal(p[2], sqrt(p[1]))` is replaced with - - -```julia -#= REPL[25]:8 =# -begin - var"##tmpright#331" = Normal.(p[2], sqrt.(p[1])) - var"##tmpright#331" isa Union{Distribution, AbstractVector{<:Distribution}} || throw(ArgumentError("Right-hand side of a ~ must be subtype of Distribution or a vector of Distributions.")) - var"##vn#333" = (DynamicPPL.VarName)(:x, ((1:2,),)) - var"##inds#334" = ((1:2,),) - var"##isassumption#335" = begin - let var"##vn#336" = (DynamicPPL.VarName)(:x, ((1:2,),)) - if !((DynamicPPL.inargnames)(var"##vn#336", _model)) || (DynamicPPL.inmissings)(var"##vn#336", _model) - true - else - x[1:2] === missing - end - end - end - if var"##isassumption#335" - x[1:2] .= (DynamicPPL.dot_tilde_assume)(_rng, _context, _sampler, var"##tmpright#331", x[1:2], var"##vn#333", var"##inds#334", _varinfo) - else - (DynamicPPL.dot_tilde_observe)(_context, _sampler, var"##tmpright#331", x[1:2], var"##vn#333", var"##inds#334", _varinfo) - end -end -``` - - -The main difference in the expanded code between `L ~ R` and `@. L ~ R` is that the former doesn't assume `L` to be defined, it can be a new Julia variable in the scope, while the latter assumes `L` already exists. Moreover, `DynamicPPL.dot_tilde_assume` and `DynamicPPL.dot_tilde_observe` are called instead of `DynamicPPL.tilde_assume` and `DynamicPPL.tilde_observe`. - - - - - - -## Step 3: Replace the user-provided function body - - -Finally, we replace the user-provided function body using `DynamicPPL.build_output`. This function uses `MacroTools.combinedef` to reassemble the user-provided function with a new function body. In the modified function body an anonymous function is created whose function body was generated in step 2 above and whose arguments are - - - * a random number generator `_rng`, - * a model `_model`, - * a datastructure `_varinfo`, - * a sampler `_sampler`, - * a sampling context `_context`, - * and all positional and keyword arguments of the user-provided model function as positional arguments - - -without any default values. Finally, in the new function body a `model::Model` with this anonymous function as internal function is returned. - - - - - - -# `VarName` - - -In order to track random variables in the sampling process, `Turing` uses the struct `VarName{sym}` which acts as a random variable identifier generated at runtime. The `VarName` of a random variable is generated from the expression on the LHS of a `~` statement when the symbol on the LHS is in `P`. Every `vn::VarName{sym}` has a symbol `sym` which is the symbol of the Julia variable in the model that the random variable belongs to. For example, `x[1] ~ Normal()` will generate an instance of `VarName{:x}` assuming `x` is in `P`. Every `vn::VarName` also has a field `indexing` which stores the indices requires to access the random variable from the Julia variable indicated by `sym`. For example, `x[1] ~ Normal()` will generate a `vn::VarName{:x}` with `vn.indexing == "[1]"`. `VarName` also supports hierarchical arrays and range indexing. Some more examples: - - - * `x[1] ~ Normal()` will generate a `VarName{:x}` with `indexing == "[1]"`. - * `x[:,1] ~ MvNormal(zeros(2))` will generate a `VarName{:x}` with `indexing == "[Colon(),1]"`. - * `x[:,1][2] ~ Normal()` will generate a `VarName{:x}` with `indexing == "[Colon(),1][2]"`. - - - - - - -# `VarInfo` - - - - -## Overview - - -`VarInfo` is the data structure in `Turing` that facilitates tracking random variables and certain metadata about them that are required for sampling. For instance, the distribution of every random variable is stored in `VarInfo` because we need to know the support of every random variable when sampling using HMC for example. Random variables whose distributions have a constrained support are transformed using a bijector from [Bijectors.jl](https://github.com/TuringLang/Bijectors.jl) so that the sampling happens in the unconstrained space. Different samplers require different metadata about the random variables. - - -The definition of `VarInfo` in `Turing` is: - - -``` -struct VarInfo{Tmeta, Tlogp} <: AbstractVarInfo - metadata::Tmeta - logp::Base.RefValue{Tlogp} - num_produce::Base.RefValue{Int} -end -``` - - -Based on the type of `metadata`, the `VarInfo` is either aliased `UntypedVarInfo` or `TypedVarInfo`. `metadata` can be either a subtype of the union type `Metadata` or a `NamedTuple` of multiple such subtypes. Let `vi` be an instance of `VarInfo`. If `vi isa VarInfo{<:Metadata}`, then it is called an `UntypedVarInfo`. If `vi isa VarInfo{<:NamedTuple}`, then `vi.metadata` would be a `NamedTuple` mapping each symbol in `P` to an instance of `Metadata`. `vi` would then be called a `TypedVarInfo`. The other fields of `VarInfo` include `logp` which is used to accumulate the log probability or log probability density of the variables in `P` and `D`. `num_produce` keeps track of how many observations have been made in the model so far. This is incremented when running a `~` statement when the symbol on the LHS is in `D`. - - - - - - -## `Metadata` - - -The `Metadata` struct stores some metadata about the random variables sampled. This helps query certain information about a variable such as: its distribution, which samplers sample this variable, its value and whether this value is transformed to real space or not. Let `md` be an instance of `Metadata`: - - - * `md.vns` is the vector of all `VarName` instances. Let `vn` be an arbitrary element of `md.vns` - * `md.idcs` is the dictionary that maps each `VarName` instance to its index in - - -`md.vns`, `md.ranges`, `md.dists`, `md.orders` and `md.flags`. - - - * `md.vns[md.idcs[vn]] == vn`. - * `md.dists[md.idcs[vn]]` is the distribution of `vn`. - * `md.gids[md.idcs[vn]]` is the set of algorithms used to sample `vn`. This is used in - - -the Gibbs sampling process. - - - * `md.orders[md.idcs[vn]]` is the number of `observe` statements before `vn` is sampled. - * `md.ranges[md.idcs[vn]]` is the index range of `vn` in `md.vals`. - * `md.vals[md.ranges[md.idcs[vn]]]` is the linearized vector of values of corresponding to `vn`. - * `md.flags` is a dictionary of true/false flags. `md.flags[flag][md.idcs[vn]]` is the - - -value of `flag` corresponding to `vn`. - - -Note that in order to make `md::Metadata` type stable, all the `md.vns` must have the same symbol and distribution type. However, one can have a single Julia variable, e.g. `x`, that is a matrix or a hierarchical array sampled in partitions, e.g. `x[1][:] ~ MvNormal(zeros(2), 1.0); x[2][:] ~ MvNormal(ones(2), 1.0)`. The symbol `x` can still be managed by a single `md::Metadata` without hurting the type stability since all the distributions on the RHS of `~` are of the same type. - - -However, in `Turing` models one cannot have this restriction, so we must use a type unstable `Metadata` if we want to use one `Metadata` instance for the whole model. This is what `UntypedVarInfo` does. A type unstable `Metadata` will still work but will have inferior performance. - - -To strike a balance between flexibility and performance when constructing the `spl::Sampler` instance, the model is first run by sampling the parameters in `P` from their priors using an `UntypedVarInfo`, i.e. a type unstable `Metadata` is used for all the variables. Then once all the symbols and distribution types have been identified, a `vi::TypedVarInfo` is constructed where `vi.metadata` is a `NamedTuple` mapping each symbol in `P` to a specialized instance of `Metadata`. So as long as each symbol in `P` is sampled from only one type of distributions, `vi::TypedVarInfo` will have fully concretely typed fields which brings out the peak performance of Julia. - diff --git a/_docs/for-developers/how_turing_implements_abstractmcmc.md b/_docs/for-developers/how_turing_implements_abstractmcmc.md deleted file mode 100644 index 425a7d16c..000000000 --- a/_docs/for-developers/how_turing_implements_abstractmcmc.md +++ /dev/null @@ -1,371 +0,0 @@ ---- -title: How Turing implements AbstractMCMC ---- - - - - - -# How Turing implements AbstractMCMC - - -Prerequisite: [Interface guide](https://turing.ml/dev/docs/for-developers/interface). - - - - - - -## Introduction - - -Consider the following Turing, code block: - - -```julia -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -mod = gdemo(1.5, 2) -alg = IS() -n_samples = 1000 - -chn = sample(mod, alg, n_samples) -``` - - -The function `sample` is part of the AbstractMCMC interface. As explained in the [interface guide](https://turing.ml/dev/docs/for-developers/interface), building a a sampling method that can be used by `sample` consists in overloading the structs and functions in `AbstractMCMC`. The interface guide also gives a standalone example of their implementation, [`AdvancedMH.jl`](). - - -Turing sampling methods (most of which are written [here](https://github.com/TuringLang/Turing.jl/tree/master/src/inference)) also implement `AbstractMCMC`. Turing defines a particular architecture for `AbstractMCMC` implementations, that enables working with models defined by the `@model` macro, and uses DynamicPPL as a backend. The goal of this page is to describe this architecture, and how you would go about implementing your own sampling method in Turing, using Importance Sampling as an example. I don't go into all the details: for instance, I don't address selectors or parallelism. - - -First, we explain how Importance Sampling works in the abstract. Consider the model defined in the first code block. Mathematically, it can be written: - - -$$ -\begin{align} -s &\sim \text{InverseGamma}(2, 3) \\ -m &\sim \text{Normal}(0, \sqrt{s}) \\ -x &\sim \text{Normal}(m, \sqrt{s}) \\ -y &\sim \text{Normal}(m, \sqrt{s}) -\end{align} -$$ - - -The **latent** variables are $$s$$ and $$m$$, the **observed** variables are $$x$$ and $$y$$. The model **joint** distribution $$p(s,m,x,y)$$ decomposes into the **prior** $$p(s,m)$$ and the **likelihood** $$p(x,y \mid s,m)$$. Since $$x = 1.5$$ and $$y = 2$$ are observed, the goal is to infer the **posterior** distribution $$p(s,m \mid x,y)$$. - - -Importance Sampling produces independent samples $$(s_i, m_i)$$ from the prior distribution. It also outputs unnormalized weights $$w_i = \frac {p(x,y,s_i,m_i)} {p(s_i, m_i)} = p(x,y \mid s_i, m_i)$$ such that the empirical distribution $$\frac 1 N \sum\limits_{i =1}^N \frac {w*i} {\sum\limits*{j=1}^N w_j} \delta*{(s\*i, m_i)}$$ is a good approximation of the posterior. - - - - - - -## 1. Define a `Sampler` - - -Recall the last line of the above code block: - - -```julia -chn = sample(mod, alg, n_samples) -``` - - -Here `sample` takes as arguments a **model** `mod`, an **algorithm** `alg`, and a **number of samples** `n_samples`, and returns an instance `chn` of `Chains` which can be analysed using the functions in `MCMCChains`. - - - - - - -### Models - - -To define a **model**, you declare a joint distribution on variables in the `@model` macro, and specify which variables are observed and which should be inferred, as well as the value of the observed variables. Thus, when implementing Importance Sampling, - - -```julia -mod = gdemo(1.5, 2) -``` - - -creates an instance `mod` of the struct `Model`, which corresponds to the observations of a value of `1.5` for `x`, and a value of `2` for `y`. - - -This is all handled by DynamicPPL, more specifically [here](https://github.com/TuringLang/DynamicPPL.jl/blob/master/src/model.jl). I will return to how models are used to inform sampling algorithms [below](#assumeobserve). - - - - - - -### Algorithms - - -An **algorithm** is just a sampling method: in Turing, it is a subtype of the abstract type `InferenceAlgorithm`. Defining an algorithm may require specifying a few high-level parameters. For example, "Hamiltonian Monte-Carlo" may be too vague, but "Hamiltonian Monte Carlo with 10 leapfrog steps per proposal and a stepsize of 0.01" is an algorithm. "Metropolis-Hastings" may be too vague, but "Metropolis-Hastings with proposal distribution `p`" is an algorithm. $$\epsilon$$ - - -Thus - - -```julia -stepsize = 0.01 -L = 10 -alg = HMC(stepsize, L) -``` - - -defines a Hamiltonian Monte-Carlo algorithm, an instance of `HMC`, which is a subtype of `InferenceAlgorithm`. - - -In the case of Importance Sampling, there is no need to specify additional parameters: - - -```julia -alg = IS() -``` - - -defines an Importance Sampling algorithm, an instance of `IS` which is a subtype of `InferenceAlgorithm`. - - -When creating your own Turing sampling method, you must therefore build a subtype of `InferenceAlgorithm` corresponding to your method. - - - - - - -### Samplers - - -Samplers are **not** the same as algorithms. An algorithm is a generic sampling method, a sampler is an object that stores information about how algorithm and model interact during sampling, and is modified as sampling progresses. The `Sampler` struct is defined in DynamicPPL. - - -Turing implements `AbstractMCMC`'s `AbstractSampler` with the `Sampler` struct defined in `DynamicPPL`. The most important attributes of an instance `spl` of `Sampler` are: - - - * `spl.alg`: the sampling method used, an instance of a subtype of `InferenceAlgorithm` - * `spl.state`: information about the sampling process, see [below](#States) - - -When you call `sample(mod, alg, n_samples)`, Turing first uses `model` and `alg` to build an instance `spl` of `Sampler` , then calls the native `AbstractMCMC` function `sample(mod, spl, n_samples)`. - - -When you define your own Turing sampling method, you must therefore build: - - - * a **sampler constructor** that uses a model and an algorithm to initialize an instance of `Sampler`. For Importance Sampling: - - -```julia -function Sampler(alg::IS, model::Model, s::Selector) - info = Dict{Symbol, Any}() - state = ISState(model) - return Sampler(alg, info, s, state) -end -``` - - - * a **state** struct implementing `AbstractSamplerState` corresponding to your method: we cover this in the following paragraph. - - - - - - -### States - - -The `vi` field contains all the important information about sampling: first and foremost, the values of all the samples, but also the distributions from which they are sampled, the names of model parameters, and other metadata. As we will see below, many important steps during sampling correspond to queries or updates to `spl.state.vi`. - - -By default, you can use `SamplerState`, a concrete type defined in `inference/Inference.jl`, which extends `AbstractSamplerState` and has no field except for `vi`: - - -```julia -mutable struct SamplerState{VIType<:VarInfo} <: AbstractSamplerState - vi :: VIType -end -``` - - -When doing Importance Sampling, we care not only about the values of the samples but also their weights. We will see below that the weight of each sample is also added to `spl.state.vi`. Moreover, the average $$\frac 1 N \sum\limits_{j=1}^N w_i = \frac 1 N \sum\limits_{j=1}^N p(x,y \mid s_i, m_i)$$ of the sample weights is a particularly important quantity: - - - * it is used to **normalize** the **empirical approximation** of the posterior distribution - * its logarithm is the importance sampling **estimate** of the **log evidence** $$\log p(x, y)$$ - - -To avoid having to compute it over and over again, `is.jl`defines an IS-specific concrete type `ISState` for sampler states, with an additional field `final_logevidence` containing $$\log \left( \frac 1 N \sum\limits_{j=1}^N w_i \right)$$. - - -```julia -mutable struct ISState{V<:VarInfo, F<:AbstractFloat} <: AbstractSamplerState - vi :: V - final_logevidence :: F -end - -# additional constructor -ISState(model::Model) = ISState(VarInfo(model), 0.0) -``` - - -The following diagram summarizes the hierarchy presented above. - - -![hierarchy](how_turing_implements_abstractmcmc_files/hierarchy.png) - - - - - - -## 2. Overload the functions used inside `mcmcsample` - - -A lot of the things here are method-specific. However Turing also has some functions that make it easier for you to implement these functions, for examples . - - - - - - -### Transitions - - -`AbstractMCMC` stores information corresponding to each individual sample in objects called `transition`, but does not specify what the structure of these objects could be. You could decide to implement a type `MyTransition` for transitions corresponding to the specifics of your methods. However, there are many situations in which the only information you need for each sample is: - - - * its value: $$\theta$$ - * log of the joint probability of the observed data and this sample: `lp` - - -`Inference.jl` [defines](https://github.com/TuringLang/Turing.jl/blob/master/src/inference/Inference.jl#L103) a struct `Transition`, which corresponds to this default situation - - -```julia -struct Transition{T, F<:AbstractFloat} - θ :: T - lp :: F -end -``` - - -It also [contains](https://github.com/TuringLang/Turing.jl/blob/master/src/inference/Inference.jl#L108) a constructor that builds an instance of `Transition` from an instance `spl` of `Sampler`: $$\theta$$ is `spl.state.vi` converted to a `namedtuple`, and `lp` is `getlogp(spl.state.vi)`. `is.jl` uses this default constructor at the end of the `step!` function [here](https://github.com/TuringLang/Turing.jl/blob/master/src/inference/is.jl#L58). - - - - - - -### How `sample` works - - -A crude summary, which ignores things like parallelism, is the following: - - -`sample` calls `mcmcsample`, which calls - - - * `sample_init!` to set things up - * `step!` repeatedly to produce multiple new transitions - * `sample_end!` to perform operations once all samples have been obtained - * `bundle_samples` to convert a vector of transitions into a more palatable type, for instance a `Chain`. - - -You can of course implement all of these functions, but `AbstractMCMC` as well as Turing also provide default implementations for simple cases. For instance, importance sampling uses the default implementations of `sample_init!` and `bundle_samples`, which is why you don't see code for them inside `is.jl`. - - - - - - -## 3. Overload `assume` and `observe` - - -The functions mentioned above, such as `sample_init!`, `step!`, etc., must of course use information about the model in order to generate samples! In particular, these functions may need **samples from distributions** defined in the model, or to **evaluate the density of these distributions** at some values of the corresponding parameters or observations. - - -For an example of the former, consider **Importance Sampling** as defined in `is.jl`. This implementation of Importance Sampling uses the model prior distribution as a proposal distribution, and therefore requires **samples from the prior distribution** of the model. Another example is **Approximate Bayesian Computation**, which requires multiple **samples from the model prior and likelihood distributions** in order to generate a single sample. - - -An example of the latter is the **Metropolis-Hastings** algorithm. At every step of sampling from a target posterior $$p(\theta \mid x_{\text{obs}})$$, in order to compute the acceptance ratio, you need to **evaluate the model joint density** $$p(\theta_{\text{prop}}, x_{\text{obs}})$$ with $$\theta_{\text{prop}}$$ a sample from the proposal and $$x_{\text{obs}}$$ the observed data. - - -This begs the question: how can these functions access model information during sampling? Recall that the model is stored as an instance `m` of `Model`. One of the attributes of `m` is the model evaluation function `m.f`, which is built by compiling the `@model` macro. Executing `f` runs the tilde statements of the model in order, and adds model information to the sampler (the instance of `Sampler` that stores information about the ongoing sampling process) at each step (see [here](https://turing.ml/dev/docs/for-developers/compiler) for more information about how the `@model` macro is compiled). The DynamicPPL functions `assume` and `observe` determine what kind of information to add to the sampler for every tilde statement. - - -Consider an instance `m` of `Model` and a sampler `spl`, with associated `VarInfo` `vi = spl.state.vi`. At some point during the sampling process, an AbstractMCMC function such as `step!` calls `m(vi, ...)`, which calls the model evaluation function `m.f(vi, ...)`. - - - * for every tilde statement in the `@model` macro, `m.f(vi, ...)` returns model-related information (samples, value of the model density, etc.), and adds it to `vi`. How does it do that? - - * recall that the code for `m.f(vi, ...)` is automatically generated by compilation of the `@model` macro - * for every tilde statement in the `@model` declaration, this code contains a call to `assume(vi, ...)` if the variable on the LHS of the tilde is a **model parameter to infer**, and `observe(vi, ...)` if the variable on the LHS of the tilde is an **observation** - * in the file corresponding to your sampling method (ie in `Turing.jl/src/inference/.jl`), you have **overloaded** `assume` and `observe`, so that they can modify `vi` to include the information and samples that you care about! - * at a minimum, `assume` and `observe` return the log density `lp` of the sample or observation. the model evaluation function then immediately calls `acclogp!(vi, lp)`, which adds `lp` to the value of the log joint density stored in `vi`. - - -Here's what `assume` looks like for Importance Sampling: - - -```julia -function DynamicPPL.assume(rng, spl::Sampler{<:IS}, dist::Distribution, vn::VarName, vi) - r = rand(rng, dist) - push!(vi, vn, r, dist, spl) - return r, 0 -end -``` - - -The function first generates a sample `r` from the distribution `dist` (the right hand side of the tilde statement). It then adds `r` to `vi`, and returns `r` and 0. - - -The `observe` function is even simpler: - - -```julia -function DynamicPPL.observe(spl::Sampler{<:IS}, dist::Distribution, value, vi) - return logpdf(dist, value) -end -``` - - -It simply returns the density (in the discrete case, the probability) of the observed value under the distribution `dist`. - - - - - - -## 4. Summary: Importance Sampling step by step - - -We focus on the AbstractMCMC functions that are overriden in `is.jl` and executed inside `mcmcsample`: `step!`, which is called `n_samples` times, and `sample_end!`, which is executed once after those `n_samples` iterations. - - - * During the $$i$$-th iteration, `step!` does 3 things: - - * `empty!(spl.state.vi)`: remove information about the previous sample from the sampler's `VarInfo` - * `model(rng, spl.state.vi, spl)`: call the model evaluation function - - * calls to `assume` add the samples from the prior $$s_i$$ and $$m_i$$ to `spl.state.vi` - * calls to both `assume` or `observe` are followed by the line `acclogp!(vi, lp)`, where `lp` is an output of `assume` and `observe` - * `lp` is set to 0 after `assume`, and to the value of the density at the observation after `observe` - * when all the tilde statements have been covered, `spl.state.vi.logp[]` is the sum of the `lp`, ie the likelihood $$\log p(x, y \mid s_i, m_i) = \log p(x \mid s_i, m_i) + \log p(y \mid s_i, m_i)$$ of the observations given the latent variable samples $$s_i$$ and $$m_i$$. - * `return Transition(spl)`: build a transition from the sampler, and return that transition - - * the transition's `vi` field is simply `spl.state.vi` - * the `lp` field contains the likelihood `spl.state.vi.logp[]` - * When the, `n_samples` iterations are completed, `sample_end!` fills the `final_logevidence` field of `spl.state` - - * it simply takes the logarithm of the average of the sample weights, using the log weights for numerical stability - diff --git a/_docs/for-developers/how_turing_implements_abstractmcmc_files/hierarchy.png b/_docs/for-developers/how_turing_implements_abstractmcmc_files/hierarchy.png deleted file mode 100644 index 1abeed687..000000000 Binary files a/_docs/for-developers/how_turing_implements_abstractmcmc_files/hierarchy.png and /dev/null differ diff --git a/_docs/for-developers/interface.md b/_docs/for-developers/interface.md deleted file mode 100644 index f10118cc6..000000000 --- a/_docs/for-developers/interface.md +++ /dev/null @@ -1,412 +0,0 @@ ---- -title: Interface Guide -toc: true ---- - - - - - -# The sampling interface - - -Turing implements a sampling interface (hosted at [AbstractMCMC](https://github.com/TuringLang/AbstractMCMC.jl)) that is intended to provide a common framework for Markov chain Monte Carlo samplers. The interface presents several structures and functions that one needs to overload in order to implement an interface-compatible sampler. - - -This guide will demonstrate how to implement the interface without Turing. - - - - - - -## Interface overview - - -Any implementation of an inference method that uses the AbstractMCMC interface should implement a subset of the following types and functions: - - -1. A subtype of `AbstractSampler`, defined as a mutable struct containing state information or sampler parameters. -2. A function `sample_init!` which performs any necessary set-up (default: do not perform any set-up). -3. A function `step!` which returns a transition that represents a single draw from the sampler. -4. A function `transitions_init` which returns a container for the transitions obtained from the sampler (default: return a `Vector{T}` of length `N` where `T` is the type of the transition obtained in the first step and `N` is the number of requested samples). -5. A function `transitions_save!` which saves transitions to the container (default: save the transition of iteration `i` at position `i` in the vector of transitions). -6. A function `sample_end!` which handles any sampler wrap-up (default: do not perform any wrap-up). -7. A function `bundle_samples` which accepts the container of transitions and returns a collection of samples (default: return the vector of transitions). - - -The interface methods with exclamation points are those that are intended to allow for state mutation. Any mutating function is meant to allow mutation where needed – you might use: - - - * `sample_init!` to run some kind of sampler preparation, before sampling begins. This could mutate a sampler's state. - * `step!` might mutate a sampler flag after each sample. - * `sample_end!` contains any wrap-up you might need to do. If you were sampling in a transformed space, this might be where you convert everything back to a constrained space. - - - - - - -## Why do you have an interface? - - -The motivation for the interface is to allow Julia's fantastic probabilistic programming language community to have a set of standards and common implementations so we can all thrive together. Markov chain Monte Carlo methods tend to have a very similar framework to one another, and so a common interface should help more great inference methods built in single-purpose packages to experience more use among the community. - - - - - - -## Implementing Metropolis-Hastings without Turing - - -[Metropolis-Hastings](https://en.wikipedia.org/wiki/Markov_chain_Monte_Carlo) is often the first sampling method that people are exposed to. It is a very straightforward algorithm and is accordingly the easiest to implement, so it makes for a good example. In this section, you will learn how to use the types and functions listed above to implement the Metropolis-Hastings sampler using the MCMC interface. - - -The full code for this implementation is housed in [AdvancedMH.jl](https://github.com/TuringLang/AdvancedMH.jl). - - - - - - -### Imports - - -Let's begin by importing the relevant libraries. We'll import `AbstracMCMC`, which contains the interface framework we'll fill out. We also need `Distributions` and `Random`. - - -```julia -# Import the relevant libraries. -import AbstractMCMC -using Distributions -using Random -``` - - -An interface extension (like the one we're writing right now) typically requires that you overload or implement several functions. Specifically, you should `import` the functions you intend to overload. This next code block accomplishes that. - - -From `Distributions`, we need `Sampleable`, `VariateForm`, and `ValueSupport`, three abstract types that define a distribution. Models in the interface are assumed to be subtypes of `Sampleable{VariateForm, ValueSupport}`. In this section our model is going be be extremely simple, so we will not end up using these except to make sure that the inference functions are dispatching correctly. - - - - - - -### Sampler - - -Let's begin our sampler definition by defining a sampler called `MetropolisHastings` which is a subtype of `AbstractSampler`. Correct typing is very important for proper interface implementation – if you are missing a subtype, your method may not be dispatched to when you call `sample`. - - -```julia -# Define a sampler type. -struct MetropolisHastings{T, D} <: AbstractMCMC.AbstractSampler - init_θ::T - proposal::D -end - -# Default constructors. -MetropolisHastings(init_θ::Real) = MetropolisHastings(init_θ, Normal(0,1)) -MetropolisHastings(init_θ::Vector{<:Real}) = MetropolisHastings(init_θ, MvNormal(length(init_θ),1)) -``` - - -Above, we have defined a sampler that stores the initial parameterization of the prior, and a distribution object from which proposals are drawn. You can have a struct that has no fields, and simply use it for dispatching onto the relevant functions, or you can store a large amount of state information in your sampler. - - -The general intuition for what to store in your sampler struct is that anything you may need to perform inference between samples but you don't want to store in a transition should go into the sampler struct. It's the only way you can carry non-sample related state information between `step!` calls. - - - - - - -### Model - - -Next, we need to have a model of some kind. A model is a struct that's a subtype of `AbstractModel` that contains whatever information is necessary to perform inference on your problem. In our case we want to know the mean and variance parameters for a standard Normal distribution, so we can keep our model to the log density of a Normal. - - -Note that we only have to do this because we are not yet integrating the sampler with Turing – Turing has a very sophisticated modelling engine that removes the need to define custom model structs. - - -```julia -# Define a model type. Stores the log density function. -struct DensityModel{F<:Function} <: AbstractMCMC.AbstractModel - ℓπ::F -end -``` - - - - - - -### Transition - - -The next step is to define some transition which we will return from each `step!` call. We'll keep it simple by just defining a wrapper struct that contains the parameter draws and the log density of that draw: - - -```julia -# Create a very basic Transition type, only stores the -# parameter draws and the log probability of the draw. -struct Transition{T, L} - θ::T - lp::L -end - -# Store the new draw and its log density. -Transition(model::DensityModel, θ) = Transition(θ, ℓπ(model, θ)) -``` - - -`Transition` can now store any type of parameter, whether it's a vector of draws from multiple parameters or a single univariate draw. - - - - - - -### Metropolis-Hastings - - -Now it's time to get into the actual inference. We've defined all of the core pieces we need, but we need to implement the `step!` function which actually performs inference. - - -As a refresher, Metropolis-Hastings implements a very basic algorithm: - - -1. Pick some initial state, $$\theta_0$$. -2. For $$t$$ in $$[1,N]$$, do - - a. Generate a proposal parameterization $$θ'_t \sim q(\theta'_t \mid \theta_{t-1})$$. - - b. Calculate the acceptance probability, $$\alpha = \text{min}\Big[1,\frac{\pi(θ'_t)}{\pi(\theta_{t-1})} \frac{q(θ_{t-1} \mid θ'_t)}{q(θ'_t \mid θ_{t-1})}) \Big]$$. - - c. If $$U \le α$$ where $$U \sim [0,1]$$, then $$\theta_t = \theta'_t$$. Otherwise, $$\theta_t = \theta_{t-1}$$. - - -Of course, it's much easier to do this in the log space, so the acceptance probability is more commonly written as - - -$$ -\alpha = \min\Big[\log \pi(θ'\_t) - \log \pi(θ\_{t-1}) + \log q(θ\_{t-1} \mid θ'\_t) - \log q(θ'\_t \mid θ\_{t-1}), 0\Big] -$$ - - -In interface terms, we should do the following: - - -1. Make a new transition containing a proposed sample. -2. Calculate the acceptance probability. -3. If we accept, return the new transition, otherwise, return the old one. - - - - - - -### Steps - - -The `step!` function is the function that performs the bulk of your inference. In our case, we will implement two `step!` functions – one for the very first iteration, and one for every subsequent iteration. - - -```julia -# Define the first step! function, which is called at the -# beginning of sampling. Return the initial parameter used -# to define the sampler. -function AbstractMCMC.step!( - rng::AbstractRNG, - model::DensityModel, - spl::MetropolisHastings, - N::Integer, - ::Nothing; - kwargs... -) - return Transition(model, spl.init_θ) -end -``` - - -The first `step!` function just packages up the initial parameterization inside the sampler, and returns it. We implicity accept the very first parameterization. - - -The other `step!` function performs the usual steps from Metropolis-Hastings. Included are several helper functions, `proposal` and `q`, which are designed to replicate the functions in the pseudocode above. - - - * `proposal` generates a new proposal in the form of a `Transition`, which can be univariate if the value passed in is univariate, or it can be multivariate if the `Transition` given is multivariate. Proposals use a basic `Normal` or `MvNormal` proposal distribution. - * `q` returns the log density of one parameterization conditional on another, according to the proposal distribution. - * `step!` generates a new proposal, checks the acceptance probability, and then returns either the previous transition or the proposed transition. - - -```julia -# Define a function that makes a basic proposal depending on a univariate -# parameterization or a multivariate parameterization. -propose(spl::MetropolisHastings, model::DensityModel, θ::Real) = - Transition(model, θ + rand(spl.proposal)) -propose(spl::MetropolisHastings, model::DensityModel, θ::Vector{<:Real}) = - Transition(model, θ + rand(spl.proposal)) -propose(spl::MetropolisHastings, model::DensityModel, t::Transition) = - propose(spl, model, t.θ) - -# Calculates the probability `q(θ|θcond)`, using the proposal distribution `spl.proposal`. -q(spl::MetropolisHastings, θ::Real, θcond::Real) = logpdf(spl.proposal, θ - θcond) -q(spl::MetropolisHastings, θ::Vector{<:Real}, θcond::Vector{<:Real}) = - logpdf(spl.proposal, θ - θcond) -q(spl::MetropolisHastings, t1::Transition, t2::Transition) = q(spl, t1.θ, t2.θ) - -# Calculate the density of the model given some parameterization. -ℓπ(model::DensityModel, θ) = model.ℓπ(θ) -ℓπ(model::DensityModel, t::Transition) = t.lp - -# Define the other step function. Returns a Transition containing -# either a new proposal (if accepted) or the previous proposal -# (if not accepted). -function AbstractMCMC.step!( - rng::AbstractRNG, - model::DensityModel, - spl::MetropolisHastings, - ::Integer, - θ_prev::Transition; - kwargs... -) - # Generate a new proposal. - θ = propose(spl, model, θ_prev) - - # Calculate the log acceptance probability. - α = ℓπ(model, θ) - ℓπ(model, θ_prev) + q(spl, θ_prev, θ) - q(spl, θ, θ_prev) - - # Decide whether to return the previous θ or the new one. - if log(rand(rng)) < min(α, 0.0) - return θ - else - return θ_prev - end -end -``` - - - - - - -### Chains - - -In the default implementation, `sample` just returns a vector of all transitions. If instead you would like to obtain a `Chains` object (e.g., to simplify downstream analysis), you have to implement the `bundle_samples` function as well. It accepts the vector of transitions and returns a collection of samples. Fortunately, our `Transition` is incredibly simple, and we only need to build a little bit of functionality to accept custom parameter names passed in by the user. - - -```julia -# A basic chains constructor that works with the Transition struct we defined. -function AbstractMCMC.bundle_samples( - rng::AbstractRNG, - ℓ::DensityModel, - s::MetropolisHastings, - N::Integer, - ts::Vector{<:Transition}, - chain_type::Type{Any}; - param_names=missing, - kwargs... -) - # Turn all the transitions into a vector-of-vectors. - vals = copy(reduce(hcat,[vcat(t.θ, t.lp) for t in ts])') - - # Check if we received any parameter names. - if ismissing(param_names) - param_names = ["Parameter $i" for i in 1:(length(first(vals))-1)] - end - - # Add the log density field to the parameter names. - push!(param_names, "lp") - - # Bundle everything up and return a Chains struct. - return Chains(vals, param_names, (internals=["lp"],)) -end -``` - - -All done! - - -You can even implement different output formats by implementing `bundle_samples` for different `chain_type`s, which can be provided as keyword argument to `sample`. As default `sample` uses `chain_type = Any`. - - - - - - -### Testing the implementation - - -Now that we have all the pieces, we should test the implementation by defining a model to calculate the mean and variance parameters of a Normal distribution. We can do this by constructing a target density function, providing a sample of data, and then running the sampler with `sample`. - - -```julia -# Generate a set of data from the posterior we want to estimate. -data = rand(Normal(5, 3), 30) - -# Define the components of a basic model. -insupport(θ) = θ[2] >= 0 -dist(θ) = Normal(θ[1], θ[2]) -density(θ) = insupport(θ) ? sum(logpdf.(dist(θ), data)) : -Inf - -# Construct a DensityModel. -model = DensityModel(density) - -# Set up our sampler with initial parameters. -spl = MetropolisHastings([0.0, 0.0]) - -# Sample from the posterior. -chain = sample(model, spl, 100000; param_names=["μ", "σ"]) -``` - - -If all the interface functions have been extended properly, you should get an output from `display(chain)` that looks something like this: - - -``` -Object of type Chains, with data of type 100000×3×1 Array{Float64,3} - -Iterations = 1:100000 -Thinning interval = 1 -Chains = 1 -Samples per chain = 100000 -internals = lp -parameters = μ, σ - -2-element Array{ChainDataFrame,1} - -Summary Statistics - -│ Row │ parameters │ mean │ std │ naive_se │ mcse │ ess │ r_hat │ -│ │ Symbol │ Float64 │ Float64 │ Float64 │ Float64 │ Any │ Any │ -├─────┼────────────┼─────────┼──────────┼────────────┼────────────┼─────────┼─────────┤ -│ 1 │ μ │ 5.33157 │ 0.854193 │ 0.0027012 │ 0.00893069 │ 8344.75 │ 1.00009 │ -│ 2 │ σ │ 4.54992 │ 0.632916 │ 0.00200146 │ 0.00534942 │ 14260.8 │ 1.00005 │ - -Quantiles - -│ Row │ parameters │ 2.5% │ 25.0% │ 50.0% │ 75.0% │ 97.5% │ -│ │ Symbol │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ -├─────┼────────────┼─────────┼─────────┼─────────┼─────────┼─────────┤ -│ 1 │ μ │ 3.6595 │ 4.77754 │ 5.33182 │ 5.89509 │ 6.99651 │ -│ 2 │ σ │ 3.5097 │ 4.09732 │ 4.47805 │ 4.93094 │ 5.96821 │ -``` - - -It looks like we're extremely close to our true parameters of `Normal(5,3)`, though with a fairly high variance due to the low sample size. - - - - - - -## Conclusion - - -We've seen how to implement the sampling interface for general projects. Turing's interface methods are ever-evolving, so please open an issue at [AbstractMCMC](https://github.com/TuringLang/AbstractMCMC.jl) with feature requests or problems. - diff --git a/_docs/for-developers/variational_inference.md b/_docs/for-developers/variational_inference.md deleted file mode 100644 index 1ad75f749..000000000 --- a/_docs/for-developers/variational_inference.md +++ /dev/null @@ -1,374 +0,0 @@ ---- -title: Variational Inference -toc: true ---- - - - - - -# Overview - - -In this post we'll have a look at what's know as **variational inference (VI)**, a family of *approximate* Bayesian inference methods. In particular, we will focus on one of the more standard VI methods called **Automatic Differentation Variational Inference (ADVI)**. If - - -Here we'll have a look at the theory behind VI, but if you're interested in how to use ADVI in Turing.jl, [checkout this tutorial](../../tutorials/9-variationalinference). - - - - - - -# Motivation - - -In Bayesian inference one usually specifies a model as follows: given data $$\{ x_i\}_{i = 1}^n$$, - - -$$ \begin{align*} \text{prior: } \quad z &\sim p(z) \\ - \text{likelihood:} \quad x_i &\overset{\text{i.i.d.}}{\sim} p(x \mid z) \quad \text{where} \quad i = 1, \dots, n \end{align*} $$ - - -where $$\overset{\text{i.i.d.}}{\sim}$$ denotes that the samples are identically independently distributed. Our goal in Bayesian inference is then to find the *posterior* $$ p(z \mid \{ x_i \}_{i = 1}^n) = \prod_{i=1}^{n} p(z \mid x_i) $$ In general one cannot obtain a closed form expression for $$p(z \mid \{ x_i \}_{i = 1}^n)$$, but one might still be able to *sample* from $$p(z \mid \{ x_i \}_{i = 1}^n)$$ with guarantees of converging to the target posterior $$p(z \mid \{ x_i \}_{i = 1}^n)$$ as the number of samples go to $$\infty$$, e.g. MCMC. - - -As you are hopefully already aware, Turing.jl provides a lot of different methods with asymptotic exactness guarantees that we can apply to such a problem! - - -Unfortunately, these unbiased samplers can be prohibitively expensive to run. As the model $$p$$ increases in complexity, the convergence of these unbiased samplers can slow down dramatically. Still, in the *infinite* limit, these methods should converge to the true posterior! But infinity is fairly large, like, *at least* more than 12, so this might take a while. - - -In such a case it might be desirable to sacrifice some of these asymptotic guarantees, and instead *approximate* the posterior $$p(z \mid \{ x_i \}_{i = 1}^n)$$ using some other model which we'll denote $$q(z)$$. - - -There are multiple approaches to take in this case, one of which is **variational inference (VI)**. - - - - - - -# Variational Inference (VI) - - -In VI, we're looking to approximate $$p(z \mid \{ x_i \}_{i = 1}^n )$$ using some *approximate* or *variational* posterior $$q(z)$$. - - -To approximate something you need a notion of what "close" means. In the context of probability densities a standard such "measure" of closeness is the *Kullback-Leibler (KL) divergence* , though this is far from the only one. The KL-divergence is defined between two densities $$q(z)$$ and $$p(z \mid \{ x_i \}_{i = 1}^n)$$ as - - -$$ \begin{align*} \mathrm{D_{KL}} \big( q(z), p(z \mid \{ x_i \}_{i = 1}^n) \big) &= \int \log \bigg( \frac{q(z)}{\prod_{i = 1}^n p(z \mid x_i)} \bigg) q(z) \mathrm{d}{z} \\ - &= \mathbb{E}_{z \sim q(z)} \big[ \log q(z) - \sum_{i = 1}^n \log p(z \mid x_i) \big] \\ - &= \mathbb{E}_{z \sim q(z)} \big[ \log q(z) \big] - \sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(z \mid x_i) \big] \end{align*} $$ - - -It's worth noting that unfortunately the KL-divergence is *not* a metric/distance in the analysis-sense due to its lack of symmetry. On the other hand, it turns out that minimizing the KL-divergence that it's actually equivalent to maximizing the log-likelihood! Also, under reasonable restrictions on the densities at hand, - - -$$ \mathrm{D_{KL}}\big(q(z), p(z \mid \{ x_i \}_{i = 1}^n) \big) = 0 \quad \iff \quad q(z) = p(z \mid \{ x_i \}_{i = 1}^n), \quad \forall z $$ - - -Therefore one could (and we will) attempt to approximate $$p(z \mid \{ x_i \}_{i = 1}^n)$$ using a density $$q(z)$$ by minimizing the KL-divergence between these two! - - -One can also show that $$\mathrm{D_{KL}} \ge 0$$, which we'll need later. Finally notice that the KL-divergence is only well-defined when in fact $$q(z)$$ is zero everywhere $$p(z \mid \{ x_i \}_{i = 1}^n)$$ is zero, i.e. - - -$$ \mathrm{supp}\big(q(z)\big) \subseteq \mathrm{supp}\big(p(z \mid x)\big) $$ - - -Otherwise there might be a point $$z_0 \sim q(z)$$ such that $$p(z_0 \mid \{ x_i \}_{i = 1}^n) = 0$$, resulting in $$\log\big(\frac{q(z)}{0}\big)$$ which doesn't make sense! - - -One major problem: as we can see in the definition of the KL-divergence, we need $$p(z \mid \{ x_i \}_{i = 1}^n)$$ for any $$z$$ if we want to compute the KL-divergence between this and $$q(z)$$. We don't have that. The entire reason we even do Bayesian inference is that we don't know the posterior! Cleary this isn't going to work. *Or is it?!* - - - - - - -## Computing KL-divergence without knowing the posterior - - -First off, recall that - - -$$ p(z \mid x_i) = \frac{p(x_i, z)}{p(x_i)} $$ - - -so we can write - - -$$ \begin{align*} \mathrm{D_{KL}} \big( q(z), p(z \mid \{ x_i \}_{i = 1}^n) \big) &= \mathbb{E}_{z \sim q(z)} \big[ \log q(z) \big] - \sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(x_i, z) - \log p(x_i) \big] \\ - &= \mathbb{E}_{z \sim q(z)} \big[ \log q(z) \big] - \sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(x_i, z) \big] + \sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(x_i) \big] \\ &= \mathbb{E}_{z \sim q(z)} \big[ \log q(z) \big] - \sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(x_i, z) \big] + \sum_{i = 1}^n \log p(x_i) \end{align*} $$ - - -where in the last equality we used the fact that $$p(x_i)$$ is independent of $$z$$. - - -Now you're probably thinking "Oh great! Now you've introduced $$p(x_i)$$ which we *also* can't compute (in general)!". Woah. Calm down human. Let's do some more algebra. The above expression can be rearranged to - - -$$ \mathrm{D_{KL}} \big( q(z), p(z \mid \{ x_i \}_{i = 1}^n) \big) + \underbrace{\sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(x_i, z) \big] - \mathbb{E}_{z \sim q(z)} \big[ \log q(z) \big]}_{=: \mathrm{ELBO}(q)} = \underbrace{\sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(x_i) \big]}_{\text{constant}} $$ - - -See? The left-hand side is *constant* and, as we mentioned before, $$\mathrm{D_{KL}} \ge 0$$. What happens if we try to *maximize* the term we just gave the completely arbitrary name $$\mathrm{ELBO}$$? Well, if $$\mathrm{ELBO}$$ goes up while $$p(x_i)$$ stays constant then $$\mathrm{D_{KL}}$$ *has to* go down! That is, the $$q(z)$$ which *minimizes* the KL-divergence is the same $$q(z)$$ which *maximizes* $$\mathrm{ELBO}(q)$$: - - -$$ \underset{q}{\mathrm{argmin}} \ \mathrm{D_{KL}} \big( q(z), p(z \mid \{ x_i \}_{i = 1}^n) \big) = \underset{q}{\mathrm{argmax}} \ \mathrm{ELBO}(q) $$ - - -where - - -$$ \begin{align*} \mathrm{ELBO}(q) &:= \bigg( \sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(x_i, z) \big] \bigg) - \mathbb{E}_{z \sim q(z)} \big[ \log q(z) \big] \\ - &= \bigg( \sum_{i = 1}^n \mathbb{E}_{z \sim q(z)} \big[ \log p(x_i, z) \big] \bigg) + \mathbb{H}\big( q(z) \big) \end{align*} $$ - - -and $$\mathbb{H} \big(q(z) \big)$$ denotes the [(differential) entropy](https://www.wikiwand.com/en/Differential_entropy) of $$q(z)$$. - - -Assuming joint $$p(x_i, z)$$ and the entropy $$\mathbb{H}\big(q(z)\big)$$ are both tractable, we can use a Monte-Carlo for the remaining expectation. This leaves us with the following tractable expression - - -$$ \underset{q}{\mathrm{argmin}} \ \mathrm{D_{KL}} \big( q(z), p(z \mid \{ x_i \}_{i = 1}^n) \big) \approx \underset{q}{\mathrm{argmax}} \ \widehat{\mathrm{ELBO}}(q) $$ - - -where - - -$$ \widehat{\mathrm{ELBO}}(q) = \frac{1}{m} \bigg( \sum_{k = 1}^m \sum_{i = 1}^n \log p(x_i, z_k) \bigg) + \mathbb{H} \big(q(z)\big) \quad \text{where} \quad z_k \sim q(z) \quad \forall k = 1, \dots, m $$ - - -Hence, as long as we can sample from $$q(z)$$ somewhat efficiently, we can indeed minimize the KL-divergence! Neat, eh? - - -Sidenote: in the case where $$q(z)$$ is tractable but $$\mathbb{H} \big(q(z) \big)$$ is *not* , we can use an Monte-Carlo estimate for this term too but this generally results in a higher-variance estimate. - - -Also, I fooled you real good: the ELBO *isn't* an arbitrary name, hah! In fact it's an abbreviation for the **expected lower bound (ELBO)** because it, uhmm, well, it's the *expected* lower bound (remember $$\mathrm{D_{KL}} \ge 0$$). Yup. - - - - - - -## Maximizing the ELBO - - -Finding the optimal $$q$$ over *all* possible densities of course isn't feasible. Instead we consider a family of *parameterized* densities $$\mathscr{D}_{\Theta}$$ where $$\Theta$$ denotes the space of possible parameters. Each density in this family $$q_{\theta} \in \mathscr{D}_{\Theta}$$ is parameterized by a unique $$\theta \in \Theta$$. Moreover, we'll assume - - -1. $$q_{\theta}(z)$$, i.e. evaluating the probability density $$q$$ at any point $$z$$, is differentiable -2. $$z \sim q_{\theta}(z)$$, i.e. the process of sampling from $$q_{\theta}(z)$$, is differentiable - - -(1) is fairly straight-forward, but (2) is a bit tricky. What does it even mean for a *sampling process* to be differentiable? This is quite an interesting problem in its own right and would require something like a [50-page paper to properly review the different approaches (highly recommended read)](https://arxiv.org/abs/1906.10652). - - -We're going to make use of a particular such approach which goes under a bunch of different names: *reparametrization trick*, *path derivative*, etc. This refers to making the assumption that all elements $$q_{\theta} \in \mathscr{Q}_{\Theta}$$ can be considered as reparameterizations of some base density, say $$\bar{q}(z)$$. That is, if $$q_{\theta} \in \mathscr{Q}_{\Theta}$$ then - - -$$ z \sim q_{\theta}(z) \quad \iff \quad z := g_{\theta}(\tilde{z}) \quad \text{where} \quad \bar{z} \sim \bar{q}(z) $$ - - -for some function $$g_{\theta}$$ differentiable wrt. $$\theta$$. So all $$q_{\theta} \in \mathscr{Q}_{\Theta}$$ are using the *same* reparameterization-function $$g$$ but each $$q_{\theta}$$ correspond to different choices of $$\theta$$ for $$f_{\theta}$$. - - -Under this assumption we can differentiate the sampling process by taking the derivative of $$g_{\theta}$$ wrt. $$\theta$$, and thus we can differentiate the entire $$\widehat{\mathrm{ELBO}}(q_{\theta})$$ wrt. $$\theta$$! With the gradient available we can either try to solve for optimality either by setting the gradient equal to zero or maximize $$\widehat{\mathrm{ELBO}}(q_{\theta})$$ stepwise by traversing $$\mathscr{Q}_{\Theta}$$ in the direction of steepest ascent. For the sake of generality, we're going to go with the stepwise approach. - - -With all this nailed down, we eventually reach the section on **Automatic Differentiation Variational Inference (ADVI)**. - - - - - - -## Automatic Differentiation Variational Inference (ADVI) - - -So let's revisit the assumptions we've made at this point: - - -1. The variational posterior $$q_{\theta}$$ is in a parameterized family of densities denoted $$\mathscr{Q}_{\Theta}$$, with $$\theta \in \Theta$$. -2. $$\mathscr{Q}_{\Theta}$$ is a space of *reparameterizable* densities with $$\bar{q}(z)$$ as the base-density. -3. The parameterization function $$g_{\theta}$$ is differentiable wrt. $$\theta$$. -4. Evaluation of the probability density $$q_{\theta}(z)$$ is differentiable wrt. $$\theta$$. -5. $$\mathbb{H}\big(q_{\theta}(z)\big)$$ is tractable. -6. Evaluation of the joint density $$p(x, z)$$ is tractable and differentiable wrt. $$z$$ -7. The support of $$p(z \mid x)$$ is a subspace of the support of $$q(z)$$: $$\mathrm{supp}\big(p(z \mid x)\big) \subseteq \mathrm{supp}\big(q(z)\big)$$. - - -All of these are not *necessary* to do VI, but they are very convenient and results in a fairly flexible approach. One distribution which has a density satisfying all of the above assumptions *except* (7) (we'll get back to this in second) for any tractable and differentiable $$p(z \mid \{ x_i \}_{i = 1}^n)$$ is the good ole' Gaussian/normal distribution: - - -$$ z \sim \mathcal{N}(\mu, \Sigma) \quad \iff \quad z = g_{\mu, L}(\bar{z}) := \mu + L^T \tilde{z} \quad \text{where} \quad \bar{z} \sim \bar{q}(z) := \mathcal{N}(1_d, I_{d \times d}) $$ - - -where $$\Sigma = L L^T$$, with $$L$$ obtained from the Cholesky-decomposition. Abusing notation a bit, we're going to write - - -$$ \theta = (\mu, \Sigma) := (\mu_1, \dots, \mu_d, L_{11}, \dots, L_{1, d}, L_{2, 1}, \dots, L_{2, d}, \dots, L_{d, 1}, \dots, L_{d, d}) $$ - - -With this assumption we finally have a tractable expression for $$\widehat{\mathrm{ELBO}}(q_{\mu, \Sigma})$$! Well, assuming (7) is holds. Since a Gaussian has non-zero probability on the entirety of $$\mathbb{R}^d$$, we also require $$p(z \mid \{ x_i \}_{i = 1}^n)$$ to have non-zero probability on all of $$\mathbb{R}^d$$. - - -Though not necessary, we'll often make a *mean-field* assumption for the variational posterior $$q(z)$$, i.e. assume independence between the latent variables. In this case, we'll write - - -$$ \theta = (\mu, \sigma^2) := (\mu_1, \dots, \mu_d, \sigma_1^2, \dots, \sigma_d^2) $$ - - - - - - -### Examples - - -As a (trivial) example we could apply the approach described above to is the following generative model for $$p(z \mid \{ x_i \}_{i = 1}^n)$$: - - -$$ \begin{align*} m &\sim \mathcal{N}(0, 1) \\ - x_i &\overset{\text{i.i.d.}}{=} \mathcal{N}(m, 1), \quad i = 1, \dots, n \end{align*} $$ - - -In this case $$z = m$$ and we have the posterior defined $$p(m \mid \{ x_i \}_{i = 1}^n) = p(m) \prod_{i = 1}^n p(x_i \mid m)$$. Then the variational posterior would be - - -$$ q_{\mu, \sigma} = \mathcal{N}(\mu, \sigma^2) \quad \text{where} \quad \mu \in \mathbb{R}, \ \sigma^2 \in \mathbb{R}^{ + } $$ - - -And since prior of $$m$$, $$\mathcal{N}(0, 1)$$, has non-zero probability on the entirety of $$\mathbb{R}$$, same as $$q(m)$$, i.e. assumption (7) above holds, everything is fine and life is good. - - -But what about this generative model for $$p(z \mid \{ x_i \}_{i = 1}^n)$$: - - -$$ \begin{align*} s &\sim \mathrm{InverseGamma}(2, 3) \\ - m &\sim \mathcal{N}(0, s) \\ - x_i &\overset{\text{i.i.d.}}{=} \mathcal{N}(m, s), \quad i = 1, \dots, n \end{align*} $$ - - -with posterior $$p(s, m \mid \{ x_i \}_{i = 1}^n) = p(s) p(m \mid s) \prod_{i = 1}^n p(x_i \mid s, m)$$ and the mean-field variational posterior $$q(s, m)$$ will be - - -$$ q_{\mu_1, \mu_2, \sigma*1^2, \sigma*2^2}(s, m) = p_{\mathcal{N}(\mu_1, \sigma_1^2)}(s) p_{\mathcal{N}(\mu_2, \sigma_2^2)}(m) $$ - - -where we've denoted the evaluation of the probability density of a Gaussian as $$p_{\mathcal{N}(\mu, \sigma^2)}(x)$$. - - -Observe that $$\mathrm{InverseGamma}(2, 3)$$ has non-zero probability only on $$\mathbb{R}^{ + } := (0, \infty)$$ which is clearly not all of $$\mathbb{R}$$ like $$q(s, m)$$ has, i.e. - - -$$ \mathrm{supp} \big( q(s, m) \big) \not\subseteq \mathrm{supp} \big( p(z \mid \{ x_i \}_{i = 1}^n) \big) $$ - - -Recall from the definition of the KL-divergence that when this is the case, the KL-divergence isn't well defined. This gets us to the *automatic* part of ADVI. - - - - - - -### "Automatic"? How? - - -For a lot of the standard (continuous) densities $$p$$ we can actually construct a probability density $$\tilde{p}$$ with non-zero probability on all of $$\mathbb{R}$$ by *transforming* the "constrained" probability density $$p$$ to $$\tilde{p}$$. In fact, in these cases this is a one-to-one relationship. As we'll see, this helps solve the support-issue we've been going on and on about. - - - - - - -#### Transforming densities using change of variables - - -If we want to compute the probability of $$x$$ taking a value in some set $$A \subseteq \mathrm{supp} \big( p(x) \big)$$, we have to integrate $$p(x)$$ over $$A$$, i.e. - - -$$ \mathbb{P}_p(x \in A) = \int_A p(x) \mathrm{d}x $$ - - -This means that if we have a differentiable bijection $$f: \mathrm{supp} \big( q(x) \big) \to \mathbb{R}^d$$ with differentiable inverse $$f^{-1}: \mathbb{R}^d \to \mathrm{supp} \big( p(x) \big)$$, we can perform a change of variables - - -$$ \mathbb{P}_p(x \in A) = \int_{f^{-1}(A)} p \big(f^{-1}(y) \big) \ \big| \det \mathcal{J}_{f^{-1}}(y) \big| \ \mathrm{d}y $$ - - -where $$\mathcal{J}_{f^{-1}}(x)$$ denotes the jacobian of $$f^{-1}$$ evaluted at $$x$$. Observe that this defines a probability distribution - - -$$ \mathbb{P}_{\tilde{p}}\big(y \in f^{-1}(A) \big) = \int_{f^{-1}(A)} \tilde{p}(y) \mathrm{d}y $$ - - -since $$f^{-1}\big(\mathrm{supp} (p(x)) \big) = \mathbb{R}^d$$ which has probability 1. This probability distribution has *density* $$\tilde{p}(y)$$ with $$\mathrm{supp} \big( \tilde{p}(y) \big) = \mathbb{R}^d$$, defined - - -$$ \tilde{p}(y) = p \big( f^{-1}(y) \big) \ \big| \det \mathcal{J}_{f^{-1}}(y) \big| $$ - - -or equivalently - - -$$ \tilde{p} \big( f(x) \big) = \frac{p(x)}{\big| \det \mathcal{J}_{f}(x) \big|} $$ - - -due to the fact that - - -$$ \big| \det \mathcal{J}_{f^{-1}}(y) \big| = \big| \det \mathcal{J}_{f}(x) \big|^{-1} $$ - - -*Note: it's also necessary that the log-abs-det-jacobian term is non-vanishing. This can for example be accomplished by assuming $$f$$ to also be elementwise monotonic.* - - - - - - -#### Back to VI - - -So why is this is useful? Well, we're looking to generalize our approach using a normal distribution to cases where the supports don't match up. How about defining $$q(z)$$ by - - -$$ \begin{align*} \eta &\sim \mathcal{N}(\mu, \Sigma) \\ - z &= f^{-1}(\eta) \end{align*} $$ - - -where $$f^{-1}: \mathbb{R}^d \to \mathrm{supp} \big( p(z \mid x) \big)$$ is a differentiable bijection with differentiable inverse. Then $$z \sim q_{\mu, \Sigma}(z) \implies z \in \mathrm{supp} \big( p(z \mid x) \big)$$ as we wanted. The resulting variational density is - - -$$ q_{\mu, \Sigma}(z) = p_{\mathcal{N}(\mu, \Sigma)}\big( f(z) \big) \ \big| \det \mathcal{J}_{f}(z) \big| $$ - - -Note that the way we've constructed $$q(z)$$ here is basically a reverse of the approach we described above. Here we sample from a distribution with support on $$\mathbb{R}$$ and transform *to* $$\mathrm{supp} \big( p(z \mid x) \big)$$. - - -If we want to write the ELBO explicitly in terms of $$\eta$$ rather than $$z$$, the first term in the ELBO becomes - - -$$ \begin{align*} \mathbb{E}_{z \sim q_{\mu, \Sigma}(z)} \big[ \log p(x_i, z) \big] &= \mathbb{E}_{\eta \sim \mathcal{N}(\mu, \Sigma)} \Bigg[ \log \frac{p\big(x_i, f^{-1}(\eta) \big)}{\big| \det \mathcal{J}_{f^{-1}}(\eta) \big|} \Bigg] \\ - &= \mathbb{E}_{\eta \sim \mathcal{N}(\mu, \Sigma)} \big[ \log p\big(x_i, f^{-1}(\eta) \big) \big] - \mathbb{E}_{\eta \sim \mathcal{N}(\mu, \Sigma)} \big[ \big| \det \mathcal{J}_{f^{-1}}(\eta) \big| \big] \end{align*} $$ - - -The entropy is invariant under change of variables, thus $$\mathbb{H} \big(q_{\mu, \Sigma}(z)\big)$$ is simply the entropy of the normal distribution which is known analytically. - - -Hence, the resulting empirical estimate of the ELBO is - - -$$ \begin{align*} \widehat{\mathrm{ELBO}}(q_{\mu, \Sigma}) &= \frac{1}{m} \bigg( \sum_{k = 1}^m \sum_{i = 1}^n \Big(\log p\big(x_i, f^{-1}(\eta_k)\big) - \log \big| \det \mathcal{J}_{f^{-1}}(\eta_k) \big| \Big) \bigg) + \mathbb{H} \big(p_{\mathcal{N}(\mu, \Sigma)}(z)\big) \\ -& \text{where} \quad z_k \sim \mathcal{N}(\mu, \Sigma) \quad \forall k = 1, \dots, m \end{align*} $$ - - -And maximizing this wrt. $$\mu$$ and $$\Sigma$$ is what's referred to as **Automatic Differentation Variational Inference (ADVI)**! - - -Now if you want to try it out, [check out the tutorial on how to use ADVI in Turing.jl](../../tutorials/9-variationalinference)! - diff --git a/_docs/library/advancedhmc.md b/_docs/library/advancedhmc.md deleted file mode 100644 index 594bae5ce..000000000 --- a/_docs/library/advancedhmc.md +++ /dev/null @@ -1,911 +0,0 @@ ---- -title: AdvancedHMC -permalink: /docs/library/advancedhmc/ -toc: true ---- - - - - - -## Index - -- [`AdvancedHMC.AbstractIntegrator`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.AbstractIntegrator) -- [`AdvancedHMC.AbstractProposal`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.AbstractProposal) -- [`AdvancedHMC.AbstractTrajectory`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.AbstractTrajectory) -- [`AdvancedHMC.AbstractTrajectorySampler`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.AbstractTrajectorySampler) -- [`AdvancedHMC.BinaryTree`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.BinaryTree) -- [`AdvancedHMC.ClassicNoUTurn`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.ClassicNoUTurn) -- [`AdvancedHMC.EndPointTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.EndPointTS) -- [`AdvancedHMC.GeneralisedNoUTurn`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.GeneralisedNoUTurn) -- [`AdvancedHMC.HMCDA`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.HMCDA) -- [`AdvancedHMC.JitteredLeapfrog`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.JitteredLeapfrog) -- [`AdvancedHMC.Leapfrog`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.Leapfrog) -- [`AdvancedHMC.MultinomialTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.MultinomialTS-Tuple{Random.AbstractRNG,AdvancedHMC.PhasePoint}) -- [`AdvancedHMC.MultinomialTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.MultinomialTS-Tuple{MultinomialTS,AbstractFloat,AdvancedHMC.PhasePoint}) -- [`AdvancedHMC.MultinomialTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.MultinomialTS) -- [`AdvancedHMC.NUTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.NUTS) -- [`AdvancedHMC.NUTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.NUTS-Tuple) -- [`AdvancedHMC.NUTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.NUTS-Union{Tuple{I}, Tuple{C}, Tuple{S}, Tuple{F}, Tuple{I}, Tuple{I,Int64}, Tuple{I,Int64,F}} where C<:AdvancedHMC.AbstractTerminationCriterion where S<:AdvancedHMC.AbstractTrajectorySampler where F<:AbstractFloat where I<:AdvancedHMC.AbstractIntegrator) -- [`AdvancedHMC.SliceTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.SliceTS) -- [`AdvancedHMC.SliceTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.SliceTS-Tuple{Random.AbstractRNG,AdvancedHMC.PhasePoint}) -- [`AdvancedHMC.SliceTS`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.SliceTS-Tuple{SliceTS,AbstractFloat,AdvancedHMC.PhasePoint}) -- [`AdvancedHMC.StaticTrajectory`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.StaticTrajectory) -- [`AdvancedHMC.StrictGeneralisedNoUTurn`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.StrictGeneralisedNoUTurn) -- [`AdvancedHMC.TemperedLeapfrog`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.TemperedLeapfrog) -- [`AdvancedHMC.Termination`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.Termination) -- [`AdvancedHMC.Termination`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.Termination-Union{Tuple{F}, Tuple{MultinomialTS,AdvancedHMC.NUTS,F,F}} where F<:AbstractFloat) -- [`AdvancedHMC.Termination`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.Termination-Union{Tuple{F}, Tuple{SliceTS,AdvancedHMC.NUTS,F,F}} where F<:AbstractFloat) -- [`AdvancedHMC.Transition`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.Transition) -- [`AdvancedHMC.A`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.A-Tuple{Any,Any,Any}) -- [`AdvancedHMC.build_tree`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.build_tree-Union{Tuple{C}, Tuple{S}, Tuple{F}, Tuple{I}, Tuple{Random.AbstractRNG,AdvancedHMC.NUTS{S,C,I,F},Hamiltonian,AdvancedHMC.PhasePoint,AdvancedHMC.AbstractTrajectorySampler,Int64,Int64,AbstractFloat}} where C<:AdvancedHMC.AbstractTerminationCriterion where S<:AdvancedHMC.AbstractTrajectorySampler where F<:AbstractFloat where I<:AdvancedHMC.AbstractIntegrator) -- [`AdvancedHMC.check_left_subtree`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.check_left_subtree-Union{Tuple{T}, Tuple{Hamiltonian,T,T,T}} where T<:(AdvancedHMC.BinaryTree{var"#s57"} where var"#s57"<:StrictGeneralisedNoUTurn)) -- [`AdvancedHMC.check_right_subtree`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.check_right_subtree-Union{Tuple{T}, Tuple{Hamiltonian,T,T,T}} where T<:(AdvancedHMC.BinaryTree{var"#s57"} where var"#s57"<:StrictGeneralisedNoUTurn)) -- [`AdvancedHMC.combine`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.combine-Tuple{AdvancedHMC.BinaryTree,AdvancedHMC.BinaryTree}) -- [`AdvancedHMC.find_good_stepsize`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.find_good_stepsize-Union{Tuple{T}, Tuple{Random.AbstractRNG,Hamiltonian,AbstractArray{T,1}}} where T<:Real) -- [`AdvancedHMC.isterminated`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.isterminated-Union{Tuple{T}, Tuple{Hamiltonian,T,T,T}} where T<:(AdvancedHMC.BinaryTree{var"#s57"} where var"#s57"<:StrictGeneralisedNoUTurn)) -- [`AdvancedHMC.isterminated`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.isterminated-Tuple{Hamiltonian,AdvancedHMC.BinaryTree{var"#s57"} where var"#s57"<:ClassicNoUTurn}) -- [`AdvancedHMC.isterminated`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.isterminated-Tuple{Hamiltonian,AdvancedHMC.BinaryTree{var"#s57"} where var"#s57"<:GeneralisedNoUTurn}) -- [`AdvancedHMC.maxabs`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.maxabs-Tuple{Any,Any}) -- [`AdvancedHMC.mh_accept_ratio`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.mh_accept_ratio-Union{Tuple{T}, Tuple{Random.AbstractRNG,T,T}} where T<:AbstractFloat) -- [`AdvancedHMC.nom_step_size`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.nom_step_size-Tuple{AdvancedHMC.AbstractIntegrator}) -- [`AdvancedHMC.pm_next!`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.pm_next!-Tuple{Any,NamedTuple}) -- [`AdvancedHMC.randcat`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.randcat-Union{Tuple{T}, Tuple{Union{Random.AbstractRNG, AbstractArray{var"#s20",1} where var"#s20"<:Random.AbstractRNG},AbstractArray{T,2}}} where T) -- [`AdvancedHMC.simple_pm_next!`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.simple_pm_next!-Tuple{Any,NamedTuple}) -- [`AdvancedHMC.stat`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.stat-Tuple{AdvancedHMC.Transition}) -- [`AdvancedHMC.step_size`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.step_size) -- [`AdvancedHMC.temper`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.temper-Tuple{TemperedLeapfrog,Any,NamedTuple{(:i, :is_half),var"#s57"} where var"#s57"<:Tuple{Integer,Bool},Int64}) -- [`AdvancedHMC.transition`]({{site.baseurl}}/docs/library/advancedhmc/#AdvancedHMC.transition-Tuple{AdvancedHMC.AbstractTrajectory,Hamiltonian,AdvancedHMC.PhasePoint}) - - - - - - -## Functions - -# -**`AdvancedHMC.A`** — *Method*. - - - -A single Hamiltonian integration step. - -NOTE: this function is intended to be used in `find_good_stepsize` only. - - -source
- -# -**`AdvancedHMC.build_tree`** — *Method*. - - - -Recursivly build a tree for a given depth `j`. - - -source
- -# -**`AdvancedHMC.check_left_subtree`** — *Method*. - - - -```julia -check_left_subtree( - h::Hamiltonian, t::T, tleft::T, tright::T -) where {T<:BinaryTree{<:StrictGeneralisedNoUTurn}} -``` - -Do a U-turn check between the leftmost phase point of `t` and the leftmost phase point of `tright`, the right subtree. - - -source
- -# -**`AdvancedHMC.check_right_subtree`** — *Method*. - - - -```julia -check_left_subtree( - h::Hamiltonian, t::T, tleft::T, tright::T -) where {T<:BinaryTree{<:StrictGeneralisedNoUTurn}} -``` - -Do a U-turn check between the rightmost phase point of `t` and the rightmost phase point of `tleft`, the left subtree. - - -source
- -# -**`AdvancedHMC.combine`** — *Method*. - - - -```julia -combine(treeleft::BinaryTree, treeright::BinaryTree) -``` - -Merge a left tree `treeleft` and a right tree `treeright` under given Hamiltonian `h`, then draw a new candidate sample and update related statistics for the resulting tree. - - -source
- -# -**`AdvancedHMC.find_good_stepsize`** — *Method*. - - - -Find a good initial leap-frog step-size via heuristic search. - - -source
- -# -**`AdvancedHMC.isterminated`** — *Method*. - - - -```julia -isterminated(h::Hamiltonian, t::BinaryTree{<:ClassicNoUTurn}) -``` - -Detect U turn for two phase points (`zleft` and `zright`) under given Hamiltonian `h` using the (original) no-U-turn cirterion. - -Ref: https://arxiv.org/abs/1111.4246, https://arxiv.org/abs/1701.02434 - - -source
- -# -**`AdvancedHMC.isterminated`** — *Method*. - - - -```julia -isterminated(h::Hamiltonian, t::BinaryTree{<:GeneralisedNoUTurn}) -``` - -Detect U turn for two phase points (`zleft` and `zright`) under given Hamiltonian `h` using the generalised no-U-turn criterion. - -Ref: https://arxiv.org/abs/1701.02434 - - -source
- -# -**`AdvancedHMC.isterminated`** — *Method*. - - - -```julia -isterminated( - h::Hamiltonian, t::T, tleft::T, tright::T -) where {T<:BinaryTree{<:StrictGeneralisedNoUTurn}} -``` - -Detect U turn for two phase points (`zleft` and `zright`) under given Hamiltonian `h` using the generalised no-U-turn criterion with additional U-turn checks. - -Ref: https://arxiv.org/abs/1701.02434 https://github.com/stan-dev/stan/pull/2800 - - -source
- -# -**`AdvancedHMC.maxabs`** — *Method*. - - - -```julia -maxabs(a, b) -``` - -Return the value with the largest absolute value. - - -source
- -# -**`AdvancedHMC.mh_accept_ratio`** — *Method*. - - - -Perform MH acceptance based on energy, i.e. negative log probability. - - -source
- -# -**`AdvancedHMC.nom_step_size`** — *Method*. - - - -```julia -nom_step_size(::AbstractIntegrator) -``` - -Get the nominal integration step size. The current integration step size may differ from this, for example if the step size is jittered. Nominal step size is usually used in adaptation. - - -source
- -# -**`AdvancedHMC.pm_next!`** — *Method*. - - - -Progress meter update with all trajectory stats, iteration number and metric shown. - - -source
- -# -**`AdvancedHMC.randcat`** — *Method*. - - - -```julia -randcat(rng, P::AbstractMatrix) -``` - -Generating Categorical random variables in a vectorized mode. `P` is supposed to be a matrix of (D, N) where each column is a probability vector. - -Example - -``` -P = [ - 0.5 0.3; - 0.4 0.6; - 0.1 0.1 -] -u = [0.3, 0.4] -C = [ - 0.5 0.3 - 0.9 0.9 - 1.0 1.0 -] -``` - -Then `C .< u'` is - -``` -[ - 0 1 - 0 0 - 0 0 -] -``` - -thus `convert.(Int, vec(sum(C .< u'; dims=1))) .+ 1` equals `[1, 2]`. - - -source
- -# -**`AdvancedHMC.simple_pm_next!`** — *Method*. - - - -Simple progress meter update without any show values. - - -source
- -# -**`AdvancedHMC.stat`** — *Method*. - - - -Returns the statistics for transition `t`. - - -source
- -# -**`AdvancedHMC.step_size`** — *Function*. - - - -```julia -step_size(::AbstractIntegrator) -``` - -Get the current integration step size. - - -source
- -# -**`AdvancedHMC.temper`** — *Method*. - - - -```julia -temper(lf::TemperedLeapfrog, r, step::NamedTuple{(:i, :is_half),<:Tuple{Integer,Bool}}, n_steps::Int) -``` - -Tempering step. `step` is a named tuple with - - * `i` being the current leapfrog iteration and - * `is_half` indicating whether or not it's (the first) half momentum/tempering step - - -source
- -# -**`AdvancedHMC.transition`** — *Method*. - - - -```julia -transition(τ::AbstractTrajectory{I}, h::Hamiltonian, z::PhasePoint) -``` - -Make a MCMC transition from phase point `z` using the trajectory `τ` under Hamiltonian `h`. - -NOTE: This is a RNG-implicit fallback function for `transition(GLOBAL_RNG, τ, h, z)` - - -source
- -# -**`StatsBase.sample`** — *Method*. - - - -```julia -sample( - rng::AbstractRNG, - h::Hamiltonian, - τ::AbstractProposal, - θ::AbstractVecOrMat{T}, - n_samples::Int, - adaptor::AbstractAdaptor=NoAdaptation(), - n_adapts::Int=min(div(n_samples, 10), 1_000); - drop_warmup::Bool=false, - verbose::Bool=true, - progress::Bool=false -) -``` - -Sample `n_samples` samples using the proposal `τ` under Hamiltonian `h`. - - * The randomness is controlled by `rng`. - - * If `rng` is not provided, `GLOBAL_RNG` will be used. - * The initial point is given by `θ`. - * The adaptor is set by `adaptor`, for which the default is no adaptation. - - * It will perform `n_adapts` steps of adaptation, for which the default is the minimum of `1_000` and 10% of `n_samples` - * `drop_warmup` controls to drop the samples during adaptation phase or not - * `verbose` controls the verbosity - * `progress` controls whether to show the progress meter or not - - -source
- - - - - - -## Types - -# -**`AdvancedHMC.AbstractIntegrator`** — *Type*. - - - -```julia -abstract type AbstractIntegrator -``` - -Represents an integrator used to simulate the Hamiltonian system. - -**Implementation** - -A `AbstractIntegrator` is expected to have the following implementations: - - * `stat`(@ref) - * `nom_step_size`(@ref) - * `step_size`(@ref) - - -source
- -# -**`AdvancedHMC.AbstractProposal`** — *Type*. - - - -Abstract Markov chain Monte Carlo proposal. - - -source
- -# -**`AdvancedHMC.AbstractTrajectory`** — *Type*. - - - -Hamiltonian dynamics numerical simulation trajectories. - - -source
- -# -**`AdvancedHMC.AbstractTrajectorySampler`** — *Type*. - - - -Defines how to sample a phase-point from the simulated trajectory. - - -source
- -# -**`AdvancedHMC.BinaryTree`** — *Type*. - - - -A full binary tree trajectory with only necessary leaves and information stored. - - -source
- -# -**`AdvancedHMC.ClassicNoUTurn`** — *Type*. - - - -```julia -struct ClassicNoUTurn <: AdvancedHMC.AbstractTerminationCriterion -``` - -Classic No-U-Turn criterion as described in Eq. (9) in [1]. - -Informally, this will terminate the trajectory expansion if continuing the simulation either forwards or backwards in time will decrease the distance between the left-most and right-most positions. - -**References** - -1. Hoffman, M. D., & Gelman, A. (2014). The No-U-Turn Sampler: adaptively setting path lengths in Hamiltonian Monte Carlo. Journal of Machine Learning Research, 15(1), 1593-1623. ([arXiv](http://arxiv.org/abs/1111.4246)) - - -source
- -# -**`AdvancedHMC.EndPointTS`** — *Type*. - - - -```julia -struct EndPointTS <: AdvancedHMC.AbstractTrajectorySampler -``` - -Samples the end-point of the trajectory. - - -source
- -# -**`AdvancedHMC.GeneralisedNoUTurn`** — *Type*. - - - -```julia -struct GeneralisedNoUTurn{T<:(AbstractArray{var"#s58",1} where var"#s58"<:Real)} <: AdvancedHMC.AbstractTerminationCriterion -``` - -Generalised No-U-Turn criterion as described in Section A.4.2 in [1]. - -**Fields** - - * `rho::AbstractArray{var"#s58",1} where var"#s58"<:Real` - - Integral or sum of momenta along the integration path. - -**References** - -1. Betancourt, M. (2017). A Conceptual Introduction to Hamiltonian Monte Carlo. [arXiv preprint arXiv:1701.02434](https://arxiv.org/abs/1701.02434). - - -source
- -# -**`AdvancedHMC.HMCDA`** — *Type*. - - - -```julia -struct HMCDA{S<:AdvancedHMC.AbstractTrajectorySampler, I<:AdvancedHMC.AbstractIntegrator} <: AdvancedHMC.DynamicTrajectory{I<:AdvancedHMC.AbstractIntegrator} -``` - -Standard HMC implementation with fixed total trajectory length. - -**Fields** - - * `integrator::AdvancedHMC.AbstractIntegrator` - - Integrator used to simulate trajectory. - * `λ::AbstractFloat` - - Total length of the trajectory, i.e. take `floor(λ / integrator_step)` number of leapfrog steps. - -**References** - -1. Neal, R. M. (2011). MCMC using Hamiltonian dynamics. Handbook of Markov chain Monte Carlo, 2(11), 2. ([arXiv](https://arxiv.org/pdf/1206.1901)) - - -source
- -# -**`AdvancedHMC.JitteredLeapfrog`** — *Type*. - - - -```julia -struct JitteredLeapfrog{FT<:AbstractFloat, T<:Union{AbstractArray{FT<:AbstractFloat,1}, FT<:AbstractFloat}} <: AdvancedHMC.AbstractLeapfrog{T<:Union{AbstractArray{FT<:AbstractFloat,1}, FT<:AbstractFloat}} -``` - -Leapfrog integrator with randomly "jittered" step size `ϵ` for every trajectory. - -**Fields** - - * `ϵ0::Union{AbstractArray{FT,1}, FT} where FT<:AbstractFloat` - - Nominal (non-jittered) step size. - * `jitter::AbstractFloat` - - The proportion of the nominal step size `ϵ0` that may be added or subtracted. - * `ϵ::Union{AbstractArray{FT,1}, FT} where FT<:AbstractFloat` - - Current (jittered) step size. - -**Description** - -This is the same as `LeapFrog`(@ref) but with a "jittered" step size. This means that at the beginning of each trajectory we sample a step size `ϵ` by adding or subtracting from the nominal/base step size `ϵ0` some random proportion of `ϵ0`, with the proportion specified by `jitter`, i.e. `ϵ = ϵ0 - jitter * ϵ0 * rand()`. p Jittering might help alleviate issues related to poor interactions with a fixed step size: - - * In regions with high "curvature" the current choice of step size might mean over-shoot leading to almost all steps being rejected. Randomly sampling the step size at the beginning of the trajectories can therefore increase the probability of escaping such high-curvature regions. - * Exact periodicity of the simulated trajectories might occur, i.e. you might be so unlucky as to simulate the trajectory forwards in time `L ϵ` and ending up at the same point (which results in non-ergodicity; see Section 3.2 in [1]). If momentum is refreshed before each trajectory, then this should not happen *exactly* but it can still be an issue in practice. Randomly choosing the step-size `ϵ` might help alleviate such problems. - -**References** - -1. Neal, R. M. (2011). MCMC using Hamiltonian dynamics. Handbook of Markov chain Monte Carlo, 2(11), 2. ([arXiv](https://arxiv.org/pdf/1206.1901)) - - -source
- -# -**`AdvancedHMC.Leapfrog`** — *Type*. - - - -```julia -struct Leapfrog{T<:(Union{AbstractArray{var"#s58",1}, var"#s58"} where var"#s58"<:AbstractFloat)} <: AdvancedHMC.AbstractLeapfrog{T<:(Union{AbstractArray{var"#s58",1}, var"#s58"} where var"#s58"<:AbstractFloat)} -``` - -Leapfrog integrator with fixed step size `ϵ`. - -**Fields** - - * `ϵ::Union{AbstractArray{var"#s58",1}, var"#s58"} where var"#s58"<:AbstractFloat` - - Step size. - - -source
- -# -**`AdvancedHMC.MultinomialTS`** — *Type*. - - - -```julia -struct MultinomialTS{F<:AbstractFloat} <: AdvancedHMC.AbstractTrajectorySampler -``` - -Multinomial trajectory sampler carried during the building of the tree. It contains the weight of the tree, defined as the total probabilities of the leaves. - -**Fields** - - * `zcand::AdvancedHMC.PhasePoint` - - Sampled candidate `PhasePoint`. - * `ℓw::AbstractFloat` - - Total energy for the given tree, i.e. the sum of energies of all leaves. - - -source
- -# -**`AdvancedHMC.MultinomialTS`** — *Method*. - - - -```julia -MultinomialTS(s::MultinomialTS, H0::AbstractFloat, zcand::PhasePoint) -``` - -Multinomial sampler for a trajectory consisting only a leaf node. - - * tree weight is the (unnormalised) energy of the leaf. - - -source
- -# -**`AdvancedHMC.MultinomialTS`** — *Method*. - - - -```julia -MultinomialTS(rng::AbstractRNG, z0::PhasePoint) -``` - -Multinomial sampler for the starting single leaf tree. (Log) weights for leaf nodes are their (unnormalised) Hamiltonian energies. - -Ref: https://github.com/stan-dev/stan/blob/develop/src/stan/mcmc/hmc/nuts/base_nuts.hpp#L226 - - -source
- -# -**`AdvancedHMC.NUTS`** — *Type*. - - - -Dynamic trajectory HMC using the no-U-turn termination criteria algorithm. - - -source
- -# -**`AdvancedHMC.NUTS`** — *Method*. - - - -```julia -NUTS(args...) = NUTS{MultinomialTS,GeneralisedNoUTurn}(args...) -``` - -Create an instance for the No-U-Turn sampling algorithm with multinomial sampling and original no U-turn criterion. - -Below is the doc for NUTS{S,C}. - -``` -NUTS{S,C}( - integrator::I, - max_depth::Int=10, - Δ_max::F=1000.0 -) where {I<:AbstractIntegrator,F<:AbstractFloat,S<:AbstractTrajectorySampler,C<:AbstractTerminationCriterion} -``` - -Create an instance for the No-U-Turn sampling algorithm. - - -source
- -# -**`AdvancedHMC.NUTS`** — *Method*. - - - -```julia -NUTS{S,C}( - integrator::I, - max_depth::Int=10, - Δ_max::F=1000.0 -) where {I<:AbstractIntegrator,F<:AbstractFloat,S<:AbstractTrajectorySampler,C<:AbstractTerminationCriterion} -``` - -Create an instance for the No-U-Turn sampling algorithm. - - -source
- -# -**`AdvancedHMC.SliceTS`** — *Type*. - - - -```julia -struct SliceTS{F<:AbstractFloat} <: AdvancedHMC.AbstractTrajectorySampler -``` - -Trajectory slice sampler carried during the building of the tree. It contains the slice variable and the number of acceptable condidates in the tree. - -**Fields** - - * `zcand::AdvancedHMC.PhasePoint` - - Sampled candidate `PhasePoint`. - * `ℓu::AbstractFloat` - - Slice variable in log-space. - * `n::Int64` - - Number of acceptable candidates, i.e. those with probability larger than slice variable `u`. - - -source
- -# -**`AdvancedHMC.SliceTS`** — *Method*. - - - -```julia -SliceTS(rng::AbstractRNG, z0::PhasePoint) -``` - -Slice sampler for the starting single leaf tree. Slice variable is initialized. - - -source
- -# -**`AdvancedHMC.SliceTS`** — *Method*. - - - -```julia -SliceTS(s::SliceTS, H0::AbstractFloat, zcand::PhasePoint) -``` - -Create a slice sampler for a single leaf tree: - - * the slice variable is copied from the passed-in sampler `s` and - * the number of acceptable candicates is computed by comparing the slice variable against the current energy. - - -source
- -# -**`AdvancedHMC.StaticTrajectory`** — *Type*. - - - -```julia -struct StaticTrajectory{S<:AdvancedHMC.AbstractTrajectorySampler, I<:AdvancedHMC.AbstractIntegrator} <: AdvancedHMC.AbstractTrajectory{I<:AdvancedHMC.AbstractIntegrator} -``` - -Static HMC with a fixed number of leapfrog steps. - -**Fields** - - * `integrator::AdvancedHMC.AbstractIntegrator` - - Integrator used to simulate trajectory. - * `n_steps::Int64` - - Number of steps to simulate, i.e. length of trajectory will be `n_steps + 1`. - -**References** - -1. Neal, R. M. (2011). MCMC using Hamiltonian dynamics. Handbook of Markov chain Monte Carlo, 2(11), 2. ([arXiv](https://arxiv.org/pdf/1206.1901)) - - -source
- -# -**`AdvancedHMC.StrictGeneralisedNoUTurn`** — *Type*. - - - -```julia -struct StrictGeneralisedNoUTurn{T<:(AbstractArray{var"#s58",1} where var"#s58"<:Real)} <: AdvancedHMC.AbstractTerminationCriterion -``` - -Generalised No-U-Turn criterion as described in Section A.4.2 in [1] with added U-turn check as described in [2]. - -**Fields** - - * `rho::AbstractArray{var"#s58",1} where var"#s58"<:Real` - - Integral or sum of momenta along the integration path. - -**References** - -1. Betancourt, M. (2017). A Conceptual Introduction to Hamiltonian Monte Carlo. [arXiv preprint arXiv:1701.02434](https://arxiv.org/abs/1701.02434). -2. [https://github.com/stan-dev/stan/pull/2800](https://github.com/stan-dev/stan/pull/2800) - - -source
- -# -**`AdvancedHMC.TemperedLeapfrog`** — *Type*. - - - -```julia -struct TemperedLeapfrog{FT<:AbstractFloat, T<:Union{AbstractArray{FT<:AbstractFloat,1}, FT<:AbstractFloat}} <: AdvancedHMC.AbstractLeapfrog{T<:Union{AbstractArray{FT<:AbstractFloat,1}, FT<:AbstractFloat}} -``` - -Tempered leapfrog integrator with fixed step size `ϵ` and "temperature" `α`. - -**Fields** - - * `ϵ::Union{AbstractArray{FT,1}, FT} where FT<:AbstractFloat` - - Step size. - * `α::AbstractFloat` - - Temperature parameter. - -**Description** - -Tempering can potentially allow greater exploration of the posterior, e.g. in a multi-modal posterior jumps between the modes can be more likely to occur. - - -source
- -# -**`AdvancedHMC.Termination`** — *Type*. - - - -```julia -Termination -``` - -Termination reasons - - * `dynamic`: due to stoping criteria - * `numerical`: due to large energy deviation from starting (possibly numerical errors) - - -source
- -# -**`AdvancedHMC.Termination`** — *Method*. - - - -```julia -Termination(s::MultinomialTS, nt::NUTS, H0::F, H′::F) where {F<:AbstractFloat} -``` - -Check termination of a Hamiltonian trajectory. - - -source
- -# -**`AdvancedHMC.Termination`** — *Method*. - - - -```julia -Termination(s::SliceTS, nt::NUTS, H0::F, H′::F) where {F<:AbstractFloat} -``` - -Check termination of a Hamiltonian trajectory. - - -source
- -# -**`AdvancedHMC.Transition`** — *Type*. - - - -```julia -struct Transition{P<:AdvancedHMC.PhasePoint, NT<:NamedTuple} -``` - -A transition that contains the phase point and other statistics of the transition. - -**Fields** - - * `z::AdvancedHMC.PhasePoint` - - Phase-point for the transition. - * `stat::NamedTuple` - - Statistics related to the transition, e.g. energy. - - -source
- diff --git a/_docs/library/api.md b/_docs/library/api.md deleted file mode 100644 index 5d4b2b9cc..000000000 --- a/_docs/library/api.md +++ /dev/null @@ -1,521 +0,0 @@ ---- -title: API -permalink: /docs/library/ -toc: true ---- - - - - - - - -## Index - -- [`Turing.BinomialLogit`]({{site.baseurl}}/docs/library/#Turing.BinomialLogit) -- [`Turing.Flat`]({{site.baseurl}}/docs/library/#Turing.Flat) -- [`Turing.FlatPos`]({{site.baseurl}}/docs/library/#Turing.FlatPos) -- [`Turing.OrderedLogistic`]({{site.baseurl}}/docs/library/#Turing.OrderedLogistic) -- [`Turing.Inference.Gibbs`]({{site.baseurl}}/docs/library/#Turing.Inference.Gibbs) -- [`Turing.Inference.HMC`]({{site.baseurl}}/docs/library/#Turing.Inference.HMC) -- [`Turing.Inference.HMCDA`]({{site.baseurl}}/docs/library/#Turing.Inference.HMCDA) -- [`Turing.Inference.IS`]({{site.baseurl}}/docs/library/#Turing.Inference.IS) -- [`Turing.Inference.MH`]({{site.baseurl}}/docs/library/#Turing.Inference.MH) -- [`Turing.Inference.NUTS`]({{site.baseurl}}/docs/library/#Turing.Inference.NUTS) -- [`Turing.Inference.PG`]({{site.baseurl}}/docs/library/#Turing.Inference.PG) -- [`Turing.Inference.SMC`]({{site.baseurl}}/docs/library/#Turing.Inference.SMC) -- [`Libtask.TArray`]({{site.baseurl}}/docs/library/#Libtask.TArray) -- [`Libtask.tzeros`]({{site.baseurl}}/docs/library/#Libtask.tzeros) - - - - - - -## Modelling - -### # **`DynamicPPL.@model`** — *Macro*. - - -```julia -@model(expr[, warn = true]) -``` - -Macro to specify a probabilistic model. - -If `warn` is `true`, a warning is displayed if internal variable names are used in the model definition. - -**Examples** - -Model definition: - -```julia -@model function model(x, y = 42) - ... -end -``` - -To generate a `Model`, call `model(xvalue)` or `model(xvalue, yvalue)`. - - - - - - -## Samplers - -### # **`DynamicPPL.Sampler`** — *Type*. - - -```julia -Sampler{T} -``` - -Generic interface for implementing inference algorithms. An implementation of an algorithm should include the following: - -1. A type specifying the algorithm and its parameters, derived from InferenceAlgorithm -2. A method of `sample` function that produces results of inference, which is where actual inference happens. - -DynamicPPL translates models to chunks that call the modelling functions at specified points. The dispatch is based on the value of a `sampler` variable. To include a new inference algorithm implements the requirements mentioned above in a separate file, then include that file at the end of this one. - -### # **`Turing.Inference.Gibbs`** — *Type*. - - -```julia -Gibbs(algs...) -``` - -Compositional MCMC interface. Gibbs sampling combines one or more sampling algorithms, each of which samples from a different set of variables in a model. - -Example: - -```julia -@model gibbs_example(x) = begin - v1 ~ Normal(0,1) - v2 ~ Categorical(5) -end -``` - -**Use PG for a 'v2' variable, and use HMC for the 'v1' variable.** - -**Note that v2 is discrete, so the PG sampler is more appropriate** - -**than is HMC.** - -alg = Gibbs(HMC(0.2, 3, :v1), PG(20, :v2)) ``` - -Tips: - - * `HMC` and `NUTS` are fast samplers, and can throw off particle-based - -methods like Particle Gibbs. You can increase the effectiveness of particle sampling by including more particles in the particle sampler. - - -source
- -### # **`Turing.Inference.HMC`** — *Type*. - - -```julia -HMC(ϵ::Float64, n_leapfrog::Int) -``` - -Hamiltonian Monte Carlo sampler with static trajectory. - -Arguments: - - * `ϵ::Float64` : The leapfrog step size to use. - * `n_leapfrog::Int` : The number of leapfrop steps to use. - -Usage: - -```julia -HMC(0.05, 10) -``` - -Tips: - - * If you are receiving gradient errors when using `HMC`, try reducing the leapfrog step size `ϵ`, e.g. - -```julia -# Original step size -sample(gdemo([1.5, 2]), HMC(0.1, 10), 1000) - -# Reduced step size -sample(gdemo([1.5, 2]), HMC(0.01, 10), 1000) -``` - - -source
- -### # **`Turing.Inference.HMCDA`** — *Type*. - - -```julia -HMCDA(n_adapts::Int, δ::Float64, λ::Float64; ϵ::Float64=0.0) -``` - -Hamiltonian Monte Carlo sampler with Dual Averaging algorithm. - -Usage: - -```julia -HMCDA(200, 0.65, 0.3) -``` - -Arguments: - - * `n_adapts::Int` : Numbers of samples to use for adaptation. - * `δ::Float64` : Target acceptance rate. 65% is often recommended. - * `λ::Float64` : Target leapfrop length. - * `ϵ::Float64=0.0` : Inital step size; 0 means automatically search by Turing. - -For more information, please view the following paper ([arXiv link](https://arxiv.org/abs/1111.4246)): - - * Hoffman, Matthew D., and Andrew Gelman. "The No-U-turn sampler: adaptively setting path lengths in Hamiltonian Monte Carlo." Journal of Machine Learning Research 15, no. 1 (2014): 1593-1623. - - -source
- -### # **`Turing.Inference.IS`** — *Type*. - - -```julia -IS() -``` - -Importance sampling algorithm. - -Note that this method is particle-based, and arrays of variables must be stored in a [`TArray`]({{site.baseurl}}/docs/library/#Libtask.TArray) object. - -Usage: - -```julia -IS() -``` - -Example: - -```julia -# Define a simple Normal model with unknown mean and variance. -@model gdemo(x) = begin - s ~ InverseGamma(2,3) - m ~ Normal(0,sqrt.(s)) - x[1] ~ Normal(m, sqrt.(s)) - x[2] ~ Normal(m, sqrt.(s)) - return s, m -end - -sample(gdemo([1.5, 2]), IS(), 1000) -``` - - -source
- -### # **`Turing.Inference.MH`** — *Type*. - - -```julia -MH(space...) -``` - -Construct a Metropolis-Hastings algorithm. - -The arguments `space` can be - - * Blank (i.e. `MH()`), in which case `MH` defaults to using the prior for each parameter as the proposal distribution. - * A set of one or more symbols to sample with `MH` in conjunction with `Gibbs`, i.e. `Gibbs(MH(:m), PG(10, :s))` - * An iterable of pairs or tuples mapping a `Symbol` to a `AdvancedMH.Proposal`, `Distribution`, or `Function` that generates returns a conditional proposal distribution. - * A covariance matrix to use as for mean-zero multivariate normal proposals. - -**Examples** - -The default `MH` will use propose samples from the prior distribution using `AdvancedMH.StaticProposal`. - -```julia -@model function gdemo(x, y) - s ~ InverseGamma(2,3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -chain = sample(gdemo(1.5, 2.0), MH(), 1_000) -mean(chain) -``` - -Alternatively, you can specify particular parameters to sample if you want to combine sampling from multiple samplers: - -`````julia -@model function gdemo(x, y) - s ~ InverseGamma(2,3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Samples s with MH and m with PG -chain = sample(gdemo(1.5, 2.0), Gibbs(MH(:s), PG(10, :m)), 1_000) -mean(chain) -```` - -Using custom distributions defaults to using static MH: - -````` - -julia @model function gdemo(x, y) s ~ InverseGamma(2,3) m ~ Normal(0, sqrt(s)) x ~ Normal(m, sqrt(s)) y ~ Normal(m, sqrt(s)) end - -**Use a static proposal for s and random walk with proposal** - -**standard deviation of 0.25 for m.** - -chain = sample( gdemo(1.5, 2.0), MH( :s => InverseGamma(2, 3), :m => Normal(0, 1) ), 1_000 ) mean(chain) - -```` - -Specifying explicit proposals using the `AdvancedMH` interface: - -```julia -@model function gdemo(x, y) - s ~ InverseGamma(2,3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Use a static proposal for s and random walk with proposal -# standard deviation of 0.25 for m. -chain = sample( - gdemo(1.5, 2.0), - MH( - :s => AdvancedMH.StaticProposal(InverseGamma(2,3)), - :m => AdvancedMH.RandomWalkProposal(Normal(0, 0.25)) - ), - 1_000 -) -mean(chain) -```` - -Using a custom function to specify a conditional distribution: - -`````julia -@model function gdemo(x, y) - s ~ InverseGamma(2,3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Use a static proposal for s and and a conditional proposal for m, -# where the proposal is centered around the current sample. -chain = sample( - gdemo(1.5, 2.0), - MH( - :s => InverseGamma(2, 3), - :m => x -> Normal(x, 1) - ), - 1_000 -) -mean(chain) -```` - -Providing a covariance matrix will cause `MH` to perform random-walk -sampling in the transformed space with proposals drawn from a multivariate -normal distribution. The provided matrix must be positive semi-definite and square. Usage: - -````` - -julia @model function gdemo(x, y) s ~ InverseGamma(2,3) m ~ Normal(0, sqrt(s)) x ~ Normal(m, sqrt(s)) y ~ Normal(m, sqrt(s)) end - -**Providing a custom variance-covariance matrix** - -chain = sample( gdemo(1.5, 2.0), MH( [0.25 0.05; 0.05 0.50] ), 1_000 ) mean(chain) ```` - - -source
- -### # **`Turing.Inference.NUTS`** — *Type*. - - -```julia -NUTS(n_adapts::Int, δ::Float64; max_depth::Int=5, Δ_max::Float64=1000.0, ϵ::Float64=0.0) -``` - -No-U-Turn Sampler (NUTS) sampler. - -Usage: - -```julia -NUTS() # Use default NUTS configuration. -NUTS(1000, 0.65) # Use 1000 adaption steps, and target accept ratio 0.65. -``` - -Arguments: - - * `n_adapts::Int` : The number of samples to use with adaptation. - * `δ::Float64` : Target acceptance rate for dual averaging. - * `max_depth::Int` : Maximum doubling tree depth. - * `Δ_max::Float64` : Maximum divergence during doubling tree. - * `ϵ::Float64` : Inital step size; 0 means automatically searching using a heuristic procedure. - - -source
- -### # **`Turing.Inference.PG`** — *Type*. - - -```julia -struct PG{space, R} <: Turing.Inference.ParticleInference -``` - -Particle Gibbs sampler. - -Note that this method is particle-based, and arrays of variables must be stored in a [`TArray`]({{site.baseurl}}/docs/library/#Libtask.TArray) object. - -**Fields** - - * `nparticles::Int64` - - Number of particles. - * `resampler::Any` - - Resampling algorithm. - - -source
- -### # **`Turing.Inference.SMC`** — *Type*. - - -```julia -struct SMC{space, R} <: Turing.Inference.ParticleInference -``` - -Sequential Monte Carlo sampler. - -**Fields** - - * `resampler::Any` - - -source
- - - - - - -## Distributions - -### # **`Turing.Flat`** — *Type*. - - -```julia -Flat <: ContinuousUnivariateDistribution -``` - -A distribution with support and density of one everywhere. - - -source
- -### # **`Turing.FlatPos`** — *Type*. - - -```julia -FlatPos(l::Real) -``` - -A distribution with a lower bound of `l` and a density of one at every `x` above `l`. - - -source
- -### # **`Turing.BinomialLogit`** — *Type*. - - -```julia -BinomialLogit(n<:Real, I<:Integer) -``` - -A univariate binomial logit distribution. - - -source
- - -!!! warning "Missing docstring." - Missing docstring for `VecBinomialLogit`. Check Documenter's build log for details. - - -### # **`Turing.OrderedLogistic`** — *Type*. - - -```julia -OrderedLogistic(η::Any, cutpoints<:AbstractVector) -``` - -An ordered logistic distribution. - - -source
- - - - - - -## Data Structures - -### # **`Libtask.TArray`** — *Type*. - - -```julia -TArray{T}(dims, ...) -``` - -Implementation of data structures that automatically perform copy-on-write after task copying. - -If current*task is an existing key in `s`, then return `s[current*task]`. Otherwise, return`s[current*task] = s[last*task]`. - -Usage: - -```julia -TArray(dim) -``` - -Example: - -```julia -ta = TArray(4) # init -for i in 1:4 ta[i] = i end # assign -Array(ta) # convert to 4-element Array{Int64,1}: [1, 2, 3, 4] -``` - - - - - - -## Utilities - -### # **`Libtask.tzeros`** — *Function*. - - -```julia - tzeros(dims, ...) -``` - -Construct a distributed array of zeros. Trailing arguments are the same as those accepted by `TArray`. - -```julia -tzeros(dim) -``` - -Example: - -```julia -tz = tzeros(4) # construct -Array(tz) # convert to 4-element Array{Int64,1}: [0, 0, 0, 0] -``` - diff --git a/_docs/library/bijectors.md b/_docs/library/bijectors.md deleted file mode 100644 index 0a2c0245f..000000000 --- a/_docs/library/bijectors.md +++ /dev/null @@ -1,612 +0,0 @@ ---- -title: Bijectors -permalink: /docs/library/bijectors/ -toc: true ---- - - - - - -## Index - -- [`Bijectors.ADBijector`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.ADBijector) -- [`Bijectors.AbstractBijector`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.AbstractBijector) -- [`Bijectors.Bijector`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.Bijector) -- [`Bijectors.Composed`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.Composed) -- [`Bijectors.CorrBijector`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.CorrBijector) -- [`Bijectors.Inverse`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.Inverse) -- [`Bijectors.Permute`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.Permute) -- [`Bijectors.Stacked`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.Stacked) -- [`Bijectors._link_chol_lkj`]({{site.baseurl}}/docs/library/bijectors/#Bijectors._link_chol_lkj-Tuple{Any}) -- [`Bijectors.bijector`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.bijector-Tuple{Distribution{Univariate,Discrete}}) -- [`Bijectors.composel`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.composel-Union{Tuple{Vararg{Bijector{N},N1} where N1}, Tuple{N}} where N) -- [`Bijectors.composer`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.composer-Union{Tuple{Vararg{Bijector{N},N1} where N1}, Tuple{N}} where N) -- [`Bijectors.compute_r`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.compute_r-Tuple{AbstractArray{var"#s107",1} where var"#s107"<:Real,Any,Any}) -- [`Bijectors.find_alpha`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.find_alpha-Tuple{AbstractArray{var"#s107",1} where var"#s107"<:Real,Any,Any,Any}) -- [`Bijectors.forward`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.forward-Tuple{Distribution}) -- [`Bijectors.forward`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.forward-Tuple{Bijector,Any}) -- [`Bijectors.isclosedform`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.isclosedform-Tuple{Bijector}) -- [`Bijectors.logabsdetjac`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.logabsdetjac-Tuple{Inverse{var"#s23",N} where N where var"#s23"<:Bijector,Any}) -- [`Bijectors.logabsdetjac`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.logabsdetjac-Tuple{ADBijector,Real}) -- [`Bijectors.logabsdetjacinv`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.logabsdetjacinv-Tuple{Bijector,Any}) -- [`Bijectors.logabsdetjacinv`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.logabsdetjacinv-Tuple{Bijectors.TransformedDistribution{var"#s108",var"#s107",Univariate} where var"#s107"<:Bijector where var"#s108"<:Distribution,Real}) -- [`Bijectors.logpdf_with_jac`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.logpdf_with_jac-Tuple{Bijectors.TransformedDistribution{var"#s108",var"#s107",Univariate} where var"#s107"<:Bijector where var"#s108"<:Distribution,Real}) -- [`Bijectors.transformed`]({{site.baseurl}}/docs/library/bijectors/#Bijectors.transformed-Tuple{Distribution,Bijector}) - - - - - - -## Functions - -# -**`Bijectors._link_chol_lkj`** — *Method*. - - - -```julia -function _link_chol_lkj(w) -``` - -Link function for cholesky factor. - -An alternative and maybe more efficient implementation was considered: - -``` -for i=2:K, j=(i+1):K - z[i, j] = (w[i, j] / w[i-1, j]) * (z[i-1, j] / sqrt(1 - z[i-1, j]^2)) -end -``` - -But this implementation will not work when w[i-1, j] = 0. Though it is a zero measure set, unit matrix initialization will not work. - -For equivelence, following explanations is given by @torfjelde: - -For `(i, j)` in the loop below, we define - -``` -z₍ᵢ₋₁, ⱼ₎ = w₍ᵢ₋₁,ⱼ₎ * ∏ₖ₌₁ⁱ⁻² (1 / √(1 - z₍ₖ,ⱼ₎²)) -``` - -and so - -``` -z₍ᵢ,ⱼ₎ = w₍ᵢ,ⱼ₎ * ∏ₖ₌₁ⁱ⁻¹ (1 / √(1 - z₍ₖ,ⱼ₎²)) - = (w₍ᵢ,ⱼ₎ * / √(1 - z₍ᵢ₋₁,ⱼ₎²)) * (∏ₖ₌₁ⁱ⁻² 1 / √(1 - z₍ₖ,ⱼ₎²)) - = (w₍ᵢ,ⱼ₎ * / √(1 - z₍ᵢ₋₁,ⱼ₎²)) * (w₍ᵢ₋₁,ⱼ₎ * ∏ₖ₌₁ⁱ⁻² 1 / √(1 - z₍ₖ,ⱼ₎²)) / w₍ᵢ₋₁,ⱼ₎ - = (w₍ᵢ,ⱼ₎ * / √(1 - z₍ᵢ₋₁,ⱼ₎²)) * (z₍ᵢ₋₁,ⱼ₎ / w₍ᵢ₋₁,ⱼ₎) - = (w₍ᵢ,ⱼ₎ / w₍ᵢ₋₁,ⱼ₎) * (z₍ᵢ₋₁,ⱼ₎ / √(1 - z₍ᵢ₋₁,ⱼ₎²)) -``` - -which is the above implementation. - -# -**`Bijectors.bijector`** — *Method*. - - - -```julia -bijector(d::Distribution) -``` - -Returns the constrained-to-unconstrained bijector for distribution `d`. - -# -**`Bijectors.composel`** — *Method*. - - - -```julia -composel(ts::Bijector...)::Composed{<:Tuple} -``` - -Constructs `Composed` such that `ts` are applied left-to-right. - -# -**`Bijectors.composer`** — *Method*. - - - -```julia -composer(ts::Bijector...)::Composed{<:Tuple} -``` - -Constructs `Composed` such that `ts` are applied right-to-left. - -# -**`Bijectors.compute_r`** — *Method*. - - - -```julia -compute_r(y_minus_z0::AbstractVector{<:Real}, α, α_plus_β_hat) -``` - -Compute the unique solution $r$ to the equation - -$$ -\|y_minus_z0\|_2 = r \left(1 + \frac{α_plus_β_hat - α}{α + r}\right) -$$ - -subject to $r ≥ 0$ and $r ≠ α$. - -Since $α > 0$ and $α_plus_β_hat > 0$, the solution is unique and given by - -$$ -r = (\sqrt{(α_plus_β_hat - γ)^2 + 4 α γ} - (α_plus_β_hat - γ)) / 2, -$$ - -where $γ = \|y_minus_z0\|_2$. For details see appendix A.2 of the reference. - -**References** - -D. Rezende, S. Mohamed (2015): Variational Inference with Normalizing Flows. arXiv:1505.05770 - -# -**`Bijectors.find_alpha`** — *Method*. - - - -```julia -find_alpha(y::AbstractVector{<:Real}, wt_y, wt_u_hat, b) -``` - -Compute an (approximate) real-valued solution $α$ to the equation - -$$ -wt_y = α + wt_u_hat tanh(α + b) -$$ - -The uniqueness of the solution is guaranteed since $wt_u_hat ≥ -1$. For details see appendix A.1 of the reference. - -**References** - -D. Rezende, S. Mohamed (2015): Variational Inference with Normalizing Flows. arXiv:1505.05770 - -# -**`Bijectors.forward`** — *Method*. - - - -```julia -forward(b::Bijector, x) -``` - -Computes both `transform` and `logabsdetjac` in one forward pass, and returns a named tuple `(rv=b(x), logabsdetjac=logabsdetjac(b, x))`. - -This defaults to the call above, but often one can re-use computation in the computation of the forward pass and the computation of the `logabsdetjac`. `forward` allows the user to take advantange of such efficiencies, if they exist. - -# -**`Bijectors.forward`** — *Method*. - - - -```julia -forward(d::Distribution) -forward(d::Distribution, num_samples::Int) -``` - -Returns a `NamedTuple` with fields `x`, `y`, `logabsdetjac` and `logpdf`. - -In the case where `d isa TransformedDistribution`, this means - - * `x = rand(d.dist)` - * `y = d.transform(x)` - * `logabsdetjac` is the logabsdetjac of the "forward" transform. - * `logpdf` is the logpdf of `y`, not `x` - -In the case where `d isa Distribution`, this means - - * `x = rand(d)` - * `y = x` - * `logabsdetjac = 0.0` - * `logpdf` is logpdf of `x` - -# -**`Bijectors.isclosedform`** — *Method*. - - - -```julia -isclosedform(b::Bijector)::bool -isclosedform(b⁻¹::Inverse{<:Bijector})::bool -``` - -Returns `true` or `false` depending on whether or not evaluation of `b` has a closed-form implementation. - -Most bijectors have closed-form evaluations, but there are cases where this is not the case. For example the *inverse* evaluation of `PlanarLayer` requires an iterative procedure to evaluate and thus is not differentiable. - -# -**`Bijectors.logabsdetjac`** — *Method*. - - - -Computes the absolute determinant of the Jacobian of the inverse-transformation. - -# -**`Bijectors.logabsdetjac`** — *Method*. - - - -```julia -logabsdetjac(b::Bijector, x) -logabsdetjac(ib::Inverse{<:Bijector}, y) -``` - -Computes the log(abs(det(J(b(x))))) where J is the jacobian of the transform. Similarily for the inverse-transform. - -Default implementation for `Inverse{<:Bijector}` is implemented as `- logabsdetjac` of original `Bijector`. - -# -**`Bijectors.logabsdetjacinv`** — *Method*. - - - -```julia -logabsdetjacinv(b::Bijector, y) -``` - -Just an alias for `logabsdetjac(inv(b), y)`. - -# -**`Bijectors.logabsdetjacinv`** — *Method*. - - - -```julia -logabsdetjacinv(td::UnivariateTransformed, y::Real) -logabsdetjacinv(td::MultivariateTransformed, y::AbstractVector{<:Real}) -``` - -Computes the `logabsdetjac` of the *inverse* transformation, since `rand(td)` returns the *transformed* random variable. - -# -**`Bijectors.logpdf_with_jac`** — *Method*. - - - -```julia -logpdf_with_jac(td::UnivariateTransformed, y::Real) -logpdf_with_jac(td::MvTransformed, y::AbstractVector{<:Real}) -logpdf_with_jac(td::MatrixTransformed, y::AbstractMatrix{<:Real}) -``` - -Makes use of the `forward` method to potentially re-use computation and returns a tuple `(logpdf, logabsdetjac)`. - -# -**`Bijectors.transformed`** — *Method*. - - - -```julia -transformed(d::Distribution) -transformed(d::Distribution, b::Bijector) -``` - -Couples distribution `d` with the bijector `b` by returning a `TransformedDistribution`. - -If no bijector is provided, i.e. `transformed(d)` is called, then `transformed(d, bijector(d))` is returned. - - - - - - -## Types - -# -**`Bijectors.ADBijector`** — *Type*. - - - -Abstract type for a `Bijector{N}` making use of auto-differentation (AD) to implement `jacobian` and, by impliciation, `logabsdetjac`. - -# -**`Bijectors.AbstractBijector`** — *Type*. - - - -Abstract type for a bijector. - -# -**`Bijectors.Bijector`** — *Type*. - - - -Abstract type of bijectors with fixed dimensionality. - -# -**`Bijectors.Composed`** — *Type*. - - - -```julia -Composed(ts::A) - -∘(b1::Bijector{N}, b2::Bijector{N})::Composed{<:Tuple} -composel(ts::Bijector{N}...)::Composed{<:Tuple} -composer(ts::Bijector{N}...)::Composed{<:Tuple} -``` - -where `A` refers to either - - * `Tuple{Vararg{<:Bijector{N}}}`: a tuple of bijectors of dimensionality `N` - * `AbstractArray{<:Bijector{N}}`: an array of bijectors of dimensionality `N` - -A `Bijector` representing composition of bijectors. `composel` and `composer` results in a `Composed` for which application occurs from left-to-right and right-to-left, respectively. - -Note that all the alternative ways of constructing a `Composed` returns a `Tuple` of bijectors. This ensures type-stability of implementations of all relating methdos, e.g. `inv`. - -If you want to use an `Array` as the container instead you can do - -``` -Composed([b1, b2, ...]) -``` - -In general this is not advised since you lose type-stability, but there might be cases where this is desired, e.g. if you have a insanely large number of bijectors to compose. - -**Examples** - -**Simple example** - -Let's consider a simple example of `Exp`: - -```julia-repl -julia> using Bijectors: Exp - -julia> b = Exp() -Exp{0}() - -julia> b ∘ b -Composed{Tuple{Exp{0},Exp{0}},0}((Exp{0}(), Exp{0}())) - -julia> (b ∘ b)(1.0) == exp(exp(1.0)) # evaluation -true - -julia> inv(b ∘ b)(exp(exp(1.0))) == 1.0 # inversion -true - -julia> logabsdetjac(b ∘ b, 1.0) # determinant of jacobian -3.718281828459045 -``` - -**Notes** - -**Order** - -It's important to note that `∘` does what is expected mathematically, which means that the bijectors are applied to the input right-to-left, e.g. first applying `b2` and then `b1`: - -```julia -(b1 ∘ b2)(x) == b1(b2(x)) # => true -``` - -But in the `Composed` struct itself, we store the bijectors left-to-right, so that - -```julia -cb1 = b1 ∘ b2 # => Composed.ts == (b2, b1) -cb2 = composel(b2, b1) # => Composed.ts == (b2, b1) -cb1(x) == cb2(x) == b1(b2(x)) # => true -``` - -**Structure** - -`∘` will result in "flatten" the composition structure while `composel` and `composer` preserve the compositional structure. This is most easily seen by an example: - -```julia-repl -julia> b = Exp() -Exp{0}() - -julia> cb1 = b ∘ b; cb2 = b ∘ b; - -julia> (cb1 ∘ cb2).ts # <= different -(Exp{0}(), Exp{0}(), Exp{0}(), Exp{0}()) - -julia> (cb1 ∘ cb2).ts isa NTuple{4, Exp{0}} -true - -julia> Bijectors.composer(cb1, cb2).ts -(Composed{Tuple{Exp{0},Exp{0}},0}((Exp{0}(), Exp{0}())), Composed{Tuple{Exp{0},Exp{0}},0}((Exp{0}(), Exp{0}()))) - -julia> Bijectors.composer(cb1, cb2).ts isa Tuple{Composed, Composed} -true -``` - -# -**`Bijectors.CorrBijector`** — *Type*. - - - -```julia -CorrBijector <: Bijector{2} -``` - -A bijector implementation of Stan's parametrization method for Correlation matrix: https://mc-stan.org/docs/2_23/reference-manual/correlation-matrix-transform-section.html - -Basically, a unconstrained strictly upper triangular matrix `y` is transformed to a correlation matrix by following readable but not that efficient form: - -``` -K = size(y, 1) -z = tanh.(y) - -for j=1:K, i=1:K - if i>j - w[i,j] = 0 - elseif 1==i==j - w[i,j] = 1 - elseif 1# -**`Bijectors.Inverse`** — *Type*. - - - -```julia -inv(b::Bijector) -Inverse(b::Bijector) -``` - -A `Bijector` representing the inverse transform of `b`. - -# -**`Bijectors.Permute`** — *Type*. - - - -```julia -Permute{A} <: Bijector{1} -``` - -A bijector implementation of a permutation. The permutation is performed using a matrix of type `A`. There are a couple of different ways to construct `Permute`: - -``` -Permute([0 1; 1 0]) # will map [1, 2] => [2, 1] -Permute([2, 1]) # will map [1, 2] => [2, 1] -Permute(2, 2 => 1, 1 => 2) # will map [1, 2] => [2, 1] -Permute(2, [1, 2] => [2, 1]) # will map [1, 2] => [2, 1] -``` - -If this is not clear, the examples might be of help. - -**Examples** - -A simple example is permuting a vector of size 3. - -```julia-repl -julia> b1 = Permute([ - 0 1 0; - 1 0 0; - 0 0 1 - ]) -Permute{Array{Int64,2}}([0 1 0; 1 0 0; 0 0 1]) - -julia> b2 = Permute([2, 1, 3]) # specify all elements at once -Permute{SparseArrays.SparseMatrixCSC{Float64,Int64}}( - - [2, 1] = 1.0 - [1, 2] = 1.0 - [3, 3] = 1.0) - -julia> b3 = Permute(3, 2 => 1, 1 => 2) # element-wise -Permute{SparseArrays.SparseMatrixCSC{Float64,Int64}}( - [2, 1] = 1.0 - [1, 2] = 1.0 - [3, 3] = 1.0) - -julia> b4 = Permute(3, [1, 2] => [2, 1]) # block-wise -Permute{SparseArrays.SparseMatrixCSC{Float64,Int64}}( - [2, 1] = 1.0 - [1, 2] = 1.0 - [3, 3] = 1.0) - -julia> b1.A == b2.A == b3.A == b4.A -true - -julia> b1([1., 2., 3.]) -3-element Array{Float64,1}: - 2.0 - 1.0 - 3.0 - -julia> b2([1., 2., 3.]) -3-element Array{Float64,1}: - 2.0 - 1.0 - 3.0 - -julia> b3([1., 2., 3.]) -3-element Array{Float64,1}: - 2.0 - 1.0 - 3.0 - -julia> b4([1., 2., 3.]) -3-element Array{Float64,1}: - 2.0 - 1.0 - 3.0 - -julia> inv(b1) -Permute{LinearAlgebra.Transpose{Int64,Array{Int64,2}}}([0 1 0; 1 0 0; 0 0 1]) - -julia> inv(b1)(b1([1., 2., 3.])) -3-element Array{Float64,1}: - 1.0 - 2.0 - 3.0 -``` - -# -**`Bijectors.Stacked`** — *Type*. - - - -```julia -Stacked(bs) -Stacked(bs, ranges) -stack(bs::Bijector{0}...) # where `0` means 0-dim `Bijector` -``` - -A `Bijector` which stacks bijectors together which can then be applied to a vector where `bs[i]::Bijector` is applied to `x[ranges[i]]::UnitRange{Int}`. - -**Arguments** - - * `bs` can be either a `Tuple` or an `AbstractArray` of 0- and/or 1-dimensional bijectors - - * If `bs` is a `Tuple`, implementations are type-stable using generated functions - * If `bs` is an `AbstractArray`, implementations are *not* type-stable and use iterative methods - * `ranges` needs to be an iterable consisting of `UnitRange{Int}` - - * `length(bs) == length(ranges)` needs to be true. - -**Examples** - -``` -b1 = Logit(0.0, 1.0) -b2 = Identity{0}() -b = stack(b1, b2) -b([0.0, 1.0]) == [b1(0.0), 1.0] # => true -``` - diff --git a/_docs/tutorials/index.md b/_docs/tutorials/index.md deleted file mode 100644 index 9619d2166..000000000 --- a/_docs/tutorials/index.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -title: Tutorials ---- - - - - - -# Tutorials - - -This section contains tutorials on how to implement common models in Turing. If you prefer to have an interactive Jupyter notebook, please fork or download the [TuringTutorials](https://github.com/TuringLang/TuringTutorials) repository. - - -A list of all the tutorials available can be found to the left. The [introduction]({{site.baseurl}}/tutorials/0-introduction) tutorial contains an introduction to coin flipping with Turing and a brief overview of probabalistic programming. - - -Tutorials are under continuous development, but there are some older version available at the TuringTutorials within the [`old-notebooks`](https://github.com/TuringLang/TuringTutorials/tree/master/old-notebooks/) section. Some of these were built using prior versions of Turing and may not function correctly, but they can assist in the syntax used for common models. - - -If there is a tutorial you would like to request, please open an issue on the [TuringTutorials](https://github.com/TuringLang/TuringTutorials) repository. - diff --git a/_docs/using-turing/advanced.md b/_docs/using-turing/advanced.md deleted file mode 100644 index 451309e3b..000000000 --- a/_docs/using-turing/advanced.md +++ /dev/null @@ -1,260 +0,0 @@ ---- -title: Advanced Usage ---- - - - - - -# Advanced Usage - - - - - - -## How to Define a Customized Distribution - - -Turing.jl supports the use of distributions from the Distributions.jl package. By extension it also supports the use of customized distributions, by defining them as subtypes of `Distribution` type of the Distributions.jl package, as well as corresponding functions. - - -Below shows a workflow of how to define a customized distribution, using our own implementation of a simple `Uniform` distribution as a simple example. - - - - - - -### 1. Define the Distribution Type - - -First, define a type of the distribution, as a subtype of a corresponding distribution type in the Distributions.jl package. - - -```julia -struct CustomUniform <: ContinuousUnivariateDistribution -end -``` - - - - - - -### 2. Implement Sampling and Evaluation of the log-pdf - - -Second, define `rand` and `logpdf`, which will be used to run the model. - - -```julia -Distributions.rand(rng::AbstractRNG, d::CustomUniform) = rand(rng) # sample in [0, 1] -Distributions.logpdf(d::CustomUniform, x::Real) = zero(x) # p(x) = 1 → logp(x) = 0 -``` - - - - - - -### 3. Define Helper Functions - - -In most cases, it may be required to define some helper functions. - - - - - - -#### 3.1 Domain Transformation - - -Certain samplers, such as `HMC`, require the domain of the priors to be unbounded. Therefore, to use our `CustomUniform` as a prior in a model we also need to define how to transform samples from `[0, 1]` to `ℝ`. To do this, we simply need to define the corresponding `Bijector` from `Bijectors.jl`, which is what `Turing.jl` uses internally to deal with constrained distributions. - - -To transform from `[0, 1]` to `ℝ` we can use the `Logit` bijector: - - -```julia -Bijectors.bijector(d::CustomUniform) = Logit(0., 1.) -``` - - -You'd do the exact same thing for `ContinuousMultivariateDistribution` and `ContinuousMatrixDistribution`. For example, `Wishart` defines a distribution over positive-definite matrices and so `bijector` returns a `PDBijector` when called with a `Wishart` distribution as an argument. For discrete distributions, there is no need to define a bijector; the `Identity` bijector is used by default. - - -Alternatively, for `UnivariateDistribution` we can define the `minimum` and `maximum` of the distribution - - -```julia -Distributions.minimum(d::CustomUniform) = 0. -Distributions.maximum(d::CustomUniform) = 1. -``` - - -and `Bijectors.jl` will return a default `Bijector` called `TruncatedBijector` which makes use of `minimum` and `maximum` derive the correct transformation. - - -Internally, Turing basically does the following when it needs to convert a constrained distribution to an unconstrained distribution, e.g. when sampling using `HMC`: - - -```julia -b = bijector(dist) -transformed_dist = transformed(dist, b) # results in distribution with transformed support + correction for logpdf -``` - - -and then we can call `rand` and `logpdf` as usual, where - - - * `rand(transformed_dist)` returns a sample in the unconstrained space, and - * `logpdf(transformed_dist, y)` returns the log density of the original distribution, but with `y` living in the unconstrained space. - - -To read more about Bijectors.jl, check out [the project README](https://github.com/TuringLang/Bijectors.jl). - - - - - - -#### 3.2 Vectorization Support - - -The vectorization syntax follows `rv ~ [distribution]`, which requires `rand` and `logpdf` to be called on multiple data points at once. An appropriate implementation for `Flat` is shown below. - - -```julia -Distributions.logpdf(d::Flat, x::AbstractVector{<:Real}) = zero(x) -``` - - - - - - -## Update the accumulated log probability in the model definition - - -Turing accumulates log probabilities internally in an internal data structure that is accessible through the internal variable `_varinfo` inside of the model definition (see below for more details about model internals). However, since users should not have to deal with internal data structures, a macro `Turing.@addlogprob!` is provided that increases the accumulated log probability. For instance, this allows you to [include arbitrary terms in the likelihood](https://github.com/TuringLang/Turing.jl/issues/1332) - - -```julia -using Turing - -myloglikelihood(x, μ) = loglikelihood(Normal(μ, 1), x) - -@model function demo(x) - μ ~ Normal() - Turing.@addlogprob! myloglikelihood(x, μ) -end -``` - - -and to [reject samples](https://github.com/TuringLang/Turing.jl/issues/1328): - - -```julia -using Turing -using LinearAlgebra - -@model function demo(x) - m ~ MvNormal(length(x)) - if dot(m, x) < 0 - Turing.@addlogprob! -Inf - # Exit the model evaluation early - return - end - - x ~ MvNormal(m, 1.0) - return -end -``` - - -Note that `@addlogprob!` always increases the accumulated log probability, regardless of the provided sampling context. For instance, if you do not want to apply `Turing.@addlogprob!` when evaluating the prior of your model but only when computing the log likelihood and the log joint probability, then you should [check the type of the internal variable `_context`](https://github.com/TuringLang/DynamicPPL.jl/issues/154) such as - - -```julia -if !isa(_context, Turing.PriorContext) - Turing.@addlogprob! myloglikelihood(x, μ) -end -``` - - - - - - -## Model Internals - - -The `@model` macro accepts a function definition and rewrites it such that call of the function generates a `Model` struct for use by the sampler. Models can be constructed by hand without the use of a macro. Taking the `gdemo` model as an example, the macro-based definition - - -```julia -using Turing - -@model function gdemo(x) - # Set priors. - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - - # Observe each value of x. - @. x ~ Normal(m, sqrt(s)) -end - -model = gdemo([1.5, 2.0]) -``` - - -is equivalent to the macro-free version - - -```julia -using Turing - -# Create the model function. -function modelf(rng, model, varinfo, sampler, context, x) - # Assume s has an InverseGamma distribution. - s = Turing.DynamicPPL.tilde_assume( - rng, - context, - sampler, - InverseGamma(2, 3), - Turing.@varname(s), - (), - varinfo, - ) - - # Assume m has a Normal distribution. - m = Turing.DynamicPPL.tilde_assume( - rng, - context, - sampler, - Normal(0, sqrt(s)), - Turing.@varname(m), - (), - varinfo, - ) - - # Observe each value of x[i] according to a Normal distribution. - Turing.DynamicPPL.dot_tilde_observe(context, sampler, Normal(m, sqrt(s)), x, varinfo) -end - -# Instantiate a Model object with our data variables. -model = Turing.Model(modelf, (x = [1.5, 2.0],)) -``` - - - - - - -## Task Copying - - -Turing [copies](https://github.com/JuliaLang/julia/issues/4085) Julia tasks to deliver efficient inference algorithms, but it also provides alternative slower implementation as a fallback. Task copying is enabled by default. Task copying requires us to use the `CTask` facility which is provided by [Libtask](https://github.com/TuringLang/Libtask.jl) to create tasks. - diff --git a/_docs/using-turing/autodiff.md b/_docs/using-turing/autodiff.md deleted file mode 100644 index 45267d755..000000000 --- a/_docs/using-turing/autodiff.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: Automatic Differentiation - ---- - - - - - -# Automatic Differentiation - - - - - - -## Switching AD Modes - - -Turing supports four packages of automatic differentiation (AD) in the back end during sampling. The default AD backend is [ForwardDiff](https://github.com/JuliaDiff/ForwardDiff.jl) for forward-mode AD. Three reverse-mode AD backends are also supported, namely [Tracker](https://github.com/FluxML/Tracker.jl), [Zygote](https://github.com/FluxML/Zygote.jl) and [ReverseDiff](https://github.com/JuliaDiff/ReverseDiff.jl). `Zygote` and `ReverseDiff` are supported optionally if explicitly loaded by the user with `using Zygote` or `using ReverseDiff` next to `using Turing`. - - -To switch between the different AD backends, one can call function `Turing.setadbackend(backend_sym)`, where `backend_sym` can be `:forwarddiff` (`ForwardDiff`), `:tracker` (`Tracker`), `:zygote` (`Zygote`) or `:reversediff` (`ReverseDiff.jl`). When using `ReverseDiff`, to compile the tape only once and cache it for later use, the user needs to load [Memoization.jl](https://github.com/marius311/Memoization.jl) first with `using Memoization` then call `Turing.setrdcache(true)`. However, note that the use of caching in certain types of models can lead to incorrect results and/or errors. Models for which the compiled tape can be safely cached are models with fixed size loops and no run-time if statements. Compile-time if statements are fine. To empty the cache, you can call `Turing.emptyrdcache()`. - - - - - - -## Compositional Sampling with Differing AD Modes - - -Turing supports intermixed automatic differentiation methods for different variable spaces. The snippet below shows using `ForwardDiff` to sample the mean (`m`) parameter, and using the Tracker-based `TrackerAD` autodiff for the variance (`s`) parameter: - - -```julia -using Turing - -# Define a simple Normal model with unknown mean and variance. -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Sample using Gibbs and varying autodiff backends. -c = sample( - gdemo(1.5, 2), - Gibbs( - HMC{Turing.ForwardDiffAD{1}}(0.1, 5, :m), - HMC{Turing.TrackerAD}(0.1, 5, :s) - ), - 1000, -) -``` - - -Generally, `TrackerAD` is faster when sampling from variables of high dimensionality (greater than 20) and `ForwardDiffAD` is more efficient for lower-dimension variables. This functionality allows those who are performance sensistive to fine tune their automatic differentiation for their specific models. - - -If the differentation method is not specified in this way, Turing will default to using whatever the global AD backend is. Currently, this defaults to `ForwardDiff`. - diff --git a/_docs/using-turing/dynamichmc.md b/_docs/using-turing/dynamichmc.md deleted file mode 100644 index 682193546..000000000 --- a/_docs/using-turing/dynamichmc.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Using DynamicHMC ---- - - - - - -# Using DynamicHMC - - -Turing supports the use of [DynamicHMC](https://github.com/tpapp/DynamicHMC.jl) as a sampler through the `DynamicNUTS` function. - - -`DynamicNUTS` is not appropriate for use in compositional inference. If you intend to use [Gibbs]({{site.baseurl}}/docs/library/#Turing.Inference.Gibbs) sampling, you must use Turing's native `NUTS` function. - - -To use the `DynamicNUTS` function, you must import the `DynamicHMC` package as well as Turing. Turing does not formally require `DynamicHMC` but will include additional functionality if both packages are present. - - -Here is a brief example of how to apply `DynamicNUTS`: - - -```julia -# Import Turing and DynamicHMC. -using LogDensityProblems, DynamicHMC, Turing - -# Model definition. -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Pull 2,000 samples using DynamicNUTS. -chn = sample(gdemo(1.5, 2.0), DynamicNUTS(), 2000) -``` - diff --git a/_docs/using-turing/gen-sampler-viz.jl b/_docs/using-turing/gen-sampler-viz.jl deleted file mode 100644 index 023c7cd98..000000000 --- a/_docs/using-turing/gen-sampler-viz.jl +++ /dev/null @@ -1,100 +0,0 @@ -ENV["GKS_ENCODING"] = "utf-8" # Allows the use of unicode characters in Plots.jl -using Plots -using StatsPlots -using Turing -using Bijectors -using Random -using DynamicPPL: getlogp, settrans!, getval, reconstruct, vectorize, setval! - -# Set a seed. -Random.seed!(0) - -# Define a strange model. -@model gdemo(x) = begin - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - bumps = sin(m) + cos(m) - m = m + 5*bumps - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end - return s, m -end - -# Define our data points. -x = [1.5, 2.0, 13.0, 2.1, 0.0] - -# Set up the model call, sample from the prior. -model = gdemo(x) -vi = Turing.VarInfo(model) - -# Convert the variance parameter to the real line before sampling. -# Note: We only have to do this here because we are being very hands-on. -# Turing will handle all of this for you during normal sampling. -dist = InverseGamma(2,3) -svn = vi.metadata.s.vns[1] -mvn = vi.metadata.m.vns[1] -setval!(vi, vectorize(dist, Bijectors.link(dist, reconstruct(dist, getval(vi, svn)))), svn) -settrans!(vi, true, svn) - -# Evaluate surface at coordinates. -function evaluate(m1, m2) - spl = Turing.SampleFromPrior() - vi[svn] = [m1] - vi[mvn] = [m2] - model(vi, spl) - getlogp(vi) -end - -function plot_sampler(chain; label="") - # Extract values from chain. - val = get(chain, [:s, :m, :lp]) - ss = link.(Ref(InverseGamma(2, 3)), val.s) - ms = val.m - lps = val.lp - - # How many surface points to sample. - granularity = 100 - - # Range start/stop points. - spread = 0.5 - σ_start = minimum(ss) - spread * std(ss); σ_stop = maximum(ss) + spread * std(ss); - μ_start = minimum(ms) - spread * std(ms); μ_stop = maximum(ms) + spread * std(ms); - σ_rng = collect(range(σ_start, stop=σ_stop, length=granularity)) - μ_rng = collect(range(μ_start, stop=μ_stop, length=granularity)) - - # Make surface plot. - p = surface(σ_rng, μ_rng, evaluate, - camera=(30, 65), - # ticks=nothing, - colorbar=false, - color=:inferno, - title=label) - - line_range = 1:length(ms) - - scatter3d!(ss[line_range], ms[line_range], lps[line_range], - mc =:viridis, marker_z=collect(line_range), msw=0, - legend=false, colorbar=false, alpha=0.5, - xlabel="σ", ylabel="μ", zlabel="Log probability", - title=label) - - return p -end; - -samplers = [ - (Gibbs(HMC(0.01, 5, :s), PG(20, :m)), "Gibbs{HMC, PG}"), - (HMC(0.01, 10), "HMC"), - (HMCDA(200, 0.65, 0.3), "HMCDA"), - (MH(), "MH()"), - (NUTS(0.65), "NUTS(0.65)"), - (NUTS(0.95), "NUTS(0.95)"), - (NUTS(0.2), "NUTS(0.2)"), - (PG(20), "PG(20)"), - (PG(50), "PG(50)")] - -for (i, (spl, spl_name)) in enumerate(samplers) - c = sample(model, spl, 1000) - p = plot_sampler(c, label="$spl_name") - savefig(joinpath(@__DIR__, "sampler-figs", "samplers-$i.svg")) -end diff --git a/_docs/using-turing/get-started.md b/_docs/using-turing/get-started.md deleted file mode 100644 index b6017253c..000000000 --- a/_docs/using-turing/get-started.md +++ /dev/null @@ -1,87 +0,0 @@ ---- -title: Getting Started ---- - - - - - -# Getting Started - - - - - - -## Installation - - -To use Turing, you need to install Julia first and then install Turing. - - - - - - -### Install Julia - - -You will need to install Julia 1.3 or greater, which you can get from [the official Julia website](http://julialang.org/downloads/). - - - - - - -### Install Turing.jl - - -Turing is an officially registered Julia package, so you can install a stable version of Turing by running the following in the Julia REPL: - - -```julia -julia> ] add Turing -``` - - -You can check if all tests pass by running - - -```julia -julia> ] test Turing -``` - - - - - - -### Example - - -Here's a simple example showing the package in action: - - -```julia -using Turing -using StatsPlots - -# Define a simple Normal model with unknown mean and variance. -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Run sampler, collect results -chn = sample(gdemo(1.5, 2), HMC(0.1, 5), 1000) - -# Summarise results -describe(chn) - -# Plot and save results -p = plot(chn) -savefig("gdemo-plot.png") -``` - diff --git a/_docs/using-turing/guide.md b/_docs/using-turing/guide.md deleted file mode 100644 index 9c0d88316..000000000 --- a/_docs/using-turing/guide.md +++ /dev/null @@ -1,773 +0,0 @@ ---- -title: Guide ---- - - - - - -# Guide - - - - - - -## Basics - - - - - - -### Introduction - - -A probabilistic program is Julia code wrapped in a `@model` macro. It can use arbitrary Julia code, but to ensure correctness of inference it should not have external effects or modify global state. Stack-allocated variables are safe, but mutable heap-allocated objects may lead to subtle bugs when using task copying. To help avoid those we provide a Turing-safe datatype `TArray` that can be used to create mutable arrays in Turing programs. - - -To specify distributions of random variables, Turing programs should use the `~` notation: - - -`x ~ distr` where `x` is a symbol and `distr` is a distribution. If `x` is undefined in the model function, inside the probabilistic program, this puts a random variable named `x`, distributed according to `distr`, in the current scope. `distr` can be a value of any type that implements `rand(distr)`, which samples a value from the distribution `distr`. If `x` is defined, this is used for conditioning in a style similar to [Anglican](https://probprog.github.io/anglican/index.html) (another PPL). In this case, `x` is an observed value, assumed to have been drawn from the distribution `distr`. The likelihood is computed using `logpdf(distr,y)`. The observe statements should be arranged so that every possible run traverses all of them in exactly the same order. This is equivalent to demanding that they are not placed inside stochastic control flow. - - -Available inference methods include Importance Sampling (IS), Sequential Monte Carlo (SMC), Particle Gibbs (PG), Hamiltonian Monte Carlo (HMC), Hamiltonian Monte Carlo with Dual Averaging (HMCDA) and The No-U-Turn Sampler (NUTS). - - - - - - -### Simple Gaussian Demo - - -Below is a simple Gaussian demo illustrate the basic usage of Turing.jl. - - -```julia -# Import packages. -using Turing -using StatsPlots - -# Define a simple Normal model with unknown mean and variance. -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end -``` - - -Note: As a sanity check, the expectation of `s` is 49/24 (2.04166666...) and the expectation of `m` is 7/6 (1.16666666...). - - -We can perform inference by using the `sample` function, the first argument of which is our probabalistic program and the second of which is a sampler. More information on each sampler is located in the [API]({{site.baseurl}}/docs/library). - - -```julia -# Run sampler, collect results. -c1 = sample(gdemo(1.5, 2), SMC(), 1000) -c2 = sample(gdemo(1.5, 2), PG(10), 1000) -c3 = sample(gdemo(1.5, 2), HMC(0.1, 5), 1000) -c4 = sample(gdemo(1.5, 2), Gibbs(PG(10, :m), HMC(0.1, 5, :s)), 1000) -c5 = sample(gdemo(1.5, 2), HMCDA(0.15, 0.65), 1000) -c6 = sample(gdemo(1.5, 2), NUTS(0.65), 1000) -``` - - -The `MCMCChains` module (which is re-exported by Turing) provides plotting tools for the `Chain` objects returned by a `sample` function. See the [MCMCChains](https://github.com/TuringLang/MCMCChains.jl) repository for more information on the suite of tools available for diagnosing MCMC chains. - - -```julia -# Summarise results -describe(c3) - -# Plot results -plot(c3) -savefig("gdemo-plot.png") -``` - - -The arguments for each sampler are: - - - * SMC: number of particles. - * PG: number of particles, number of iterations. - * HMC: leapfrog step size, leapfrog step numbers. - * Gibbs: component sampler 1, component sampler 2, ... - * HMCDA: total leapfrog length, target accept ratio. - * NUTS: number of adaptation steps (optional), target accept ratio. - - -For detailed information on the samplers, please review Turing.jl's [API]({{site.baseurl}}/docs/library) documentation. - - - - - - -### Modelling Syntax Explained - - -Using this syntax, a probabilistic model is defined in Turing. The model function generated by Turing can then be used to condition the model onto data. Subsequently, the sample function can be used to generate samples from the posterior distribution. - - -In the following example, the defined model is conditioned to the date (arg*1 = 1, arg*2 = 2) by passing (1, 2) to the model function. - - -```julia -@model function model_name(arg_1, arg_2) - ... -end -``` - - -The conditioned model can then be passed onto the sample function to run posterior inference. - - -```julia -model_func = model_name(1, 2) -chn = sample(model_func, HMC(..)) # Perform inference by sampling using HMC. -``` - - -The returned chain contains samples of the variables in the model. - - -```julia -var_1 = mean(chn[:var_1]) # Taking the mean of a variable named var_1. -``` - - -The key (`:var_1`) can be a `Symbol` or a `String`. For example, to fetch `x[1]`, one can use `chn[Symbol("x[1]")` or `chn["x[1]"]`. If you want to retrieve all parameters associated with a specific symbol, you can use `group`. As an example, if you have the parameters `"x[1]"`, `"x[2]"`, and `"x[3]"`, calling `group(chn, :x)` or `group(chn, "x")` will return a new chain with only `"x[1]"`, `"x[2]"`, and `"x[3]"`. - - -Turing does not have a declarative form. More generally, the order in which you place the lines of a `@model` macro matters. For example, the following example works: - - -```julia -# Define a simple Normal model with unknown mean and variance. -@model function model_function(y) - s ~ Poisson(1) - y ~ Normal(s, 1) - return y -end - -sample(model_function(10), SMC(), 100) -``` - - -But if we switch the `s ~ Poisson(1)` and `y ~ Normal(s, 1)` lines, the model will no longer sample correctly: - - -```julia -# Define a simple Normal model with unknown mean and variance. -@model function model_function(y) - y ~ Normal(s, 1) - s ~ Poisson(1) - return y -end - -sample(model_function(10), SMC(), 100) -``` - - - - - - -### Sampling Multiple Chains - - -Turing supports distributed and threaded parallel sampling. To do so, call `sample(model, sampler, parallel_type, n, n_chains)`, where `parallel_type` can be either `MCMCThreads()` or `MCMCDistributed()` for thread and parallel sampling, respectively. - - -Having multiple chains in the same object is valuable for evaluating convergence. Some diagnostic functions like `gelmandiag` require multiple chains. - - -If you do not want parallelism or are on an older version Julia, you can sample multiple chains with the `mapreduce` function: - - -```julia -# Replace num_chains below with however many chains you wish to sample. -chains = mapreduce(c -> sample(model_fun, sampler, 1000), chainscat, 1:num_chains) -``` - - -The `chains` variable now contains a `Chains` object which can be indexed by chain. To pull out the first chain from the `chains` object, use `chains[:,:,1]`. The method is the same if you use either of the below parallel sampling methods. - - - - - - -#### Multithreaded sampling - - -If you wish to perform multithreaded sampling and are running Julia 1.3 or greater, you can call `sample` with the following signature: - - -```julia -using Turing - -@model function gdemo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end - -model = gdemo([1.5, 2.0]) - -# Sample four chains using multiple threads, each with 1000 samples. -sample(model, NUTS(), MCMCThreads(), 1000, 4) -``` - - -Be aware that Turing cannot add threads for you – you must have started your Julia instance with multiple threads to experience any kind of parallelism. See the [Julia documentation](https://docs.julialang.org/en/v1/manual/parallel-computing/#man-multithreading-1) for details on how to achieve this. - - - - - - -#### Distributed sampling - - -To perform distributed sampling (using multiple processes), you must first import `Distributed`. - - -Process parallel sampling can be done like so: - - -```julia -# Load Distributed to add processes and the @everywhere macro. -using Distributed - -# Load Turing. -using Turing - -# Add four processes to use for sampling. -addprocs(4) - -# Initialize everything on all the processes. -# Note: Make sure to do this after you've already loaded Turing, -# so each process does not have to precompile. -# Parallel sampling may fail silently if you do not do this. -@everywhere using Turing - -# Define a model on all processes. -@everywhere @model function gdemo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end - -# Declare the model instance everywhere. -@everywhere model = gdemo([1.5, 2.0]) - -# Sample four chains using multiple processes, each with 1000 samples. -sample(model, NUTS(), MCMCDistributed(), 1000, 4) -``` - - - - - - -### Sampling from an Unconditional Distribution (The Prior) - - -Turing allows you to sample from a declared model's prior. If you wish to draw a chain from the prior to inspect your prior distributions, you can simply run - - -```julia -chain = sample(model, Prior(), n_samples) -``` - - -You can also run your model (as if it were a function) from the prior distribution, by calling the model without specifying inputs or a sampler. In the below example, we specify a `gdemo` model which returns two variables, `x` and `y`. The model includes `x` and `y` as arguments, but calling the function without passing in `x` or `y` means that Turing's compiler will assume they are missing values to draw from the relevant distribution. The `return` statement is necessary to retrieve the sampled `x` and `y` values. - - -```julia -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) - return x, y -end -``` - - -Assign the function with `missing` inputs to a variable, and Turing will produce a sample from the prior distribution. - - -```julia -# Samples from p(x,y) -g_prior_sample = gdemo(missing, missing) -g_prior_sample() -``` - - -Output: - - -``` -(0.685690547873451, -1.1972706455914328) -``` - - - - - - -### Sampling from a Conditional Distribution (The Posterior) - - - - - - -#### Treating observations as random variables - - -Inputs to the model that have a value `missing` are treated as parameters, aka random variables, to be estimated/sampled. This can be useful if you want to simulate draws for that parameter, or if you are sampling from a conditional distribution. Turing supports the following syntax: - - -```julia -@model function gdemo(x, ::Type{T} = Float64) where {T} - if x === missing - # Initialize `x` if missing - x = Vector{T}(undef, 2) - end - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end - -# Construct a model with x = missing -model = gdemo(missing) -c = sample(model, HMC(0.01, 5), 500) -``` - - -Note the need to initialize `x` when missing since we are iterating over its elements later in the model. The generated values for `x` can be extracted from the `Chains` object using `c[:x]`. - - -Turing also supports mixed `missing` and non-`missing` values in `x`, where the missing ones will be treated as random variables to be sampled while the others get treated as observations. For example: - - -```julia -@model function gdemo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end - -# x[1] is a parameter, but x[2] is an observation -model = gdemo([missing, 2.4]) -c = sample(model, HMC(0.01, 5), 500) -``` - - - - - - -#### Default Values - - -Arguments to Turing models can have default values much like how default values work in normal Julia functions. For instance, the following will assign `missing` to `x` and treat it as a random variable. If the default value is not `missing`, `x` will be assigned that value and will be treated as an observation instead. - - -```julia -using Turing - -@model function generative(x = missing, ::Type{T} = Float64) where {T <: Real} - if x === missing - # Initialize x when missing - x = Vector{T}(undef, 10) - end - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - for i in 1:length(x) - x[i] ~ Normal(m, sqrt(s)) - end - return s, m -end - -m = generative() -chain = sample(m, HMC(0.01, 5), 1000) -``` - - - - - - -#### Access Values inside Chain - - -You can access the values inside a chain several ways: - - -1. Turn them into a `DataFrame` object -2. Use their raw `AxisArray` form -3. Create a three-dimensional `Array` object - - -For example, let `c` be a `Chain`: 1. `DataFrame(c)` converts `c` to a `DataFrame`, 2. `c.value` retrieves the values inside `c` as an `AxisArray`, and 3. `c.value.data` retrieves the values inside `c` as a 3D `Array`. - - - - - - -#### Variable Types and Type Parameters - - -The element type of a vector (or matrix) of random variables should match the `eltype` of the its prior distribution, `<: Integer` for discrete distributions and `<: AbstractFloat` for continuous distributions. Moreover, if the continuous random variable is to be sampled using a Hamiltonian sampler, the vector's element type needs to either be: 1. `Real` to enable auto-differentiation through the model which uses special number types that are sub-types of `Real`, or 2. Some type parameter `T` defined in the model header using the type parameter syntax, e.g. `gdemo(x, ::Type{T} = Float64) where {T} = begin`. Similarly, when using a particle sampler, the Julia variable used should either be: 1. A `TArray`, or 2. An instance of some type parameter `T` defined in the model header using the type parameter syntax, e.g. `gdemo(x, ::Type{T} = Vector{Float64}) where {T} = begin`. - - - - - - -### Querying Probabilities from Model or Chain - - -Consider the following `gdemo` model: - - -```julia -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end -``` - - -The following are examples of valid queries of the `Turing` model or chain: - - - * `prob"x = 1.0, y = 1.0 | model = gdemo, s = 1.0, m = 1.0"` calculates the likelihood of `x = 1` and `y = 1` given `s = 1` and `m = 1`. - * `prob"s = 1.0, m = 1.0 | model = gdemo, x = nothing, y = nothing"` calculates the joint probability of `s = 1` and `m = 1` ignoring `x` and `y`. `x` and `y` are ignored so they can be optionally dropped from the RHS of `|`, but it is recommended to define them. - * `prob"s = 1.0, m = 1.0, x = 1.0 | model = gdemo, y = nothing"` calculates the joint probability of `s = 1`, `m = 1` and `x = 1` ignoring `y`. - * `prob"s = 1.0, m = 1.0, x = 1.0, y = 1.0 | model = gdemo"` calculates the joint probability of all the variables. - * After the MCMC sampling, given a `chain`, `prob"x = 1.0, y = 1.0 | chain = chain, model = gdemo"` calculates the element-wise likelihood of `x = 1.0` and `y = 1.0` for each sample in `chain`. - * If `save_state=true` was used during sampling (i.e., `sample(model, sampler, N; save_state=true)`), you can simply do `prob"x = 1.0, y = 1.0 | chain = chain"`. - - -In all the above cases, `logprob` can be used instead of `prob` to calculate the log probabilities instead. - - - - - - -### Maximum likelihood and maximum a posterior estimates - - -Turing provides support for two mode estimation techniques, [maximum likelihood estimation](https://en.wikipedia.org/wiki/Maximum_likelihood_estimation) (MLE) and [maximum a posterior](https://en.wikipedia.org/wiki/Maximum_a_posteriori_estimation) (MAP) estimation. Optimization is performed by the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) package. Mode estimation is currently a optional tool, and will not be available to you unless you have manually installed Optim and loaded the package with a `using` statement. To install Optim, run `import Pkg; Pkg.add("Optim")`. - - -Mode estimation only works when all model parameters are continuous – discrete parameters cannot be estimated with MLE/MAP as of yet. - - -To understand how mode estimation works, let us first load Turing and Optim to enable mode estimation, and then declare a model: - - -```julia -# Note that loading Optim explicitly is required for mode estimation to function, -# as Turing does not load the opimization suite unless Optim is loaded as well. -using Turing -using Optim - -@model function gdemo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end -``` - - -Once the model is defined, we can construct a model instance as we normally would: - - -```julia -# Create some data to pass to the model. -data = [1.5, 2.0] - -# Instantiate the gdemo model with our data. -model = gdemo(data) -``` - - -Mode estimation is typically quick and easy at this point. Turing extends the function `Optim.optimize` and accepts the structs `MLE()` or `MAP()`, which inform Turing whether to provide an MLE or MAP estimate, respectively. By default, the [LBFGS optimizer](https://julianlsolvers.github.io/Optim.jl/stable/#algo/lbfgs/) is used, though this can be changed. Basic usage is: - - -```julia -# Generate a MLE estimate. -mle_estimate = optimize(model, MLE()) - -# Generate a MAP estimate. -map_estimate = optimize(model, MAP()) -``` - - -If you wish to change to a different optimizer, such as `NelderMead`, simply place your optimizer in the third argument slot: - - -```julia -# Use NelderMead -mle_estimate = optimize(model, MLE(), NelderMead()) - -# Use SimulatedAnnealing -mle_estimate = optimize(model, MLE(), SimulatedAnnealing()) - -# Use ParticleSwarm -mle_estimate = optimize(model, MLE(), ParticleSwarm()) - -# Use Newton -mle_estimate = optimize(model, MLE(), Newton()) - -# Use AcceleratedGradientDescent -mle_estimate = optimize(model, MLE(), AcceleratedGradientDescent()) -``` - - -Some methods may have trouble calculating the mode because not enough iterations were allowed, or the target function moved upwards between function calls. Turing will warn you if Optim fails to converge by running `Optim.converge`. A typical solution to this might be to add more iterations, or allow the optimizer to increase between function iterations: - - -```julia -# Increase the iterations and allow function eval to increase between calls. -mle_estimate = optimize(model, MLE(), Newton(), Optim.Options(iterations=10_000, allow_f_increases=true)) -``` - - -More options for Optim are available [here](https://julianlsolvers.github.io/Optim.jl/stable/#user/config/). - - - - - - -#### Analyzing your mode estimate - - -Turing extends several methods from `StatsBase` that can be used to analyze your mode estimation results. Methods implemented include `vcov`, `informationmatrix`, `coeftable`, `params`, and `coef`, among others. - - -For example, let's examine our ML estimate from above using `coeftable`: - - -```julia -# Import StatsBase to use it's statistical methods. -using StatsBase - -# Print out the coefficient table. -coeftable(mle_estimate) -``` - - -```julia -───────────────────────────── - estimate stderror tstat -───────────────────────────── -s 0.0625 0.0625 1.0 -m 1.75 0.176777 9.8995 -───────────────────────────── -``` - - -Standard errors are calculated from the Fisher information matrix (inverse Hessian of the log likelihood or log joint). t-statistics will be familiar to frequentist statisticians. Warning – standard errors calculated in this way may not always be appropriate for MAP estimates, so please be cautious in interpreting them. - - - - - - -#### Sampling with the MAP/MLE as initial states - - -You can begin sampling your chain from an MLE/MAP estimate by extracting the vector of parameter values and providing it to the `sample` function with the keyword `init_theta`. For example, here is how to sample from the full posterior using the MAP estimate as the starting point: - - -```julia -# Generate an MAP estimate. -map_estimate = optimize(model, MAP()) - -# Sample with the MAP estimate as the starting point. -chain = sample(model, NUTS(), 1_000, init_theta = map_estimate.values.array) -``` - - - - - - -## Beyond the Basics - - - - - - -### Compositional Sampling Using Gibbs - - -Turing.jl provides a Gibbs interface to combine different samplers. For example, one can combine an `HMC` sampler with a `PG` sampler to run inference for different parameters in a single model as below. - - -```julia -@model function simple_choice(xs) - p ~ Beta(2, 2) - z ~ Bernoulli(p) - for i in 1:length(xs) - if z == 1 - xs[i] ~ Normal(0, 1) - else - xs[i] ~ Normal(2, 1) - end - end -end - -simple_choice_f = simple_choice([1.5, 2.0, 0.3]) - -chn = sample(simple_choice_f, Gibbs(HMC(0.2, 3, :p), PG(20, :z)), 1000) -``` - - -The `Gibbs` sampler can be used to specify unique automatic differentation backends for different variable spaces. Please see the [Automatic Differentiation]({{site.baseurl}}/docs/using-turing/autodiff) article for more. - - -For more details of compositional sampling in Turing.jl, please check the corresponding [paper](http://proceedings.mlr.press/v84/ge18b.html). - - - - - - -### Working with MCMCChains.jl - - -Turing.jl wraps its samples using `MCMCChains.Chain` so that all the functions working for `MCMCChains.Chain` can be re-used in Turing.jl. Two typical functions are `MCMCChains.describe` and `MCMCChains.plot`, which can be used as follows for an obtained chain `chn`. For more information on `MCMCChains`, please see the [GitHub repository](https://github.com/TuringLang/MCMCChains.jl). - - -```julia -describe(chn) # Lists statistics of the samples. -plot(chn) # Plots statistics of the samples. -``` - - -There are numerous functions in addition to `describe` and `plot` in the `MCMCChains` package, such as those used in convergence diagnostics. For more information on the package, please see the [GitHub repository](https://github.com/TuringLang/MCMCChains.jl). - - - - - - -### Working with Libtask.jl - - -The [Libtask.jl](https://github.com/TuringLang/Libtask.jl) library provides write-on-copy data structures that are safe for use in Turing's particle-based samplers. One data structure in particular is often required for use – the [`TArray`](http://turing.ml/docs/library/#Libtask.TArray). The following sampler types require the use of a `TArray` to store distributions: - - - * `IPMCMC` - * `IS` - * `PG` - * `PMMH` - * `SMC` - - -If you do not use a `TArray` to store arrays of distributions when using a particle-based sampler, you may experience errors. - - -Here is an example of how the `TArray` (using a `TArray` constructor function called `tzeros`) can be applied in this way: - - -```julia -# Turing model definition. -@model function BayesHmm(y) - # Declare a TArray with a length of N. - s = tzeros(Int, N) - m = Vector{Real}(undef, K) - T = Vector{Vector{Real}}(undef, K) - for i = 1:K - T[i] ~ Dirichlet(ones(K)/K) - m[i] ~ Normal(i, 0.01) - end - - # Draw from a distribution for each element in s. - s[1] ~ Categorical(K) - for i = 2:N - s[i] ~ Categorical(vec(T[s[i-1]])) - y[i] ~ Normal(m[s[i]], 0.1) - end - return (s, m) -end; -``` - - - - - - -### Changing Default Settings - - -Some of Turing.jl's default settings can be changed for better usage. - - - - - - -#### AD Chunk Size - - -ForwardDiff (Turing's default AD backend) uses forward-mode chunk-wise AD. The chunk size can be manually set by `setchunksize(new_chunk_size)`; alternatively, use an auto-tuning helper function `auto_tune_chunk_size!(mf::Function, rep_num=10)`, which will profile various chunk sizes. Here `mf` is the model function, e.g. `gdemo(1.5, 2)`, and `rep_num` is the number of repetitions during profiling. - - - - - - -#### AD Backend - - -Turing supports four packages of automatic differentiation (AD) in the back end during sampling. The default AD backend is [ForwardDiff](https://github.com/JuliaDiff/ForwardDiff.jl) for forward-mode AD. Three reverse-mode AD backends are also supported, namely [Tracker](https://github.com/FluxML/Tracker.jl), [Zygote](https://github.com/FluxML/Zygote.jl) and [ReverseDiff](https://github.com/JuliaDiff/ReverseDiff.jl). `Zygote` and `ReverseDiff` are supported optionally if explicitly loaded by the user with `using Zygote` or `using ReverseDiff` next to `using Turing`. - - -For more information on Turing's automatic differentiation backend, please see the [Automatic Differentiation]({{site.baseurl}}/docs/using-turing/autodiff) article. - - - - - - -#### Progress Logging - - -Turing.jl uses ProgressLogging.jl to log the progress of sampling. Progress logging is enabled as default but might slow down inference. It can be turned on or off by setting the keyword argument `progress` of `sample` to `true` or `false`, respectively. Moreover, you can enable or disable progress logging globally by calling `turnprogress(true)` or `turnprogress(false)`, respectively. - - -Turing uses heuristics to select an appropriate visualization backend. If you use [Juno](https://junolab.org/), the progress is displayed with a [progress bar in the Atom window](http://docs.junolab.org/latest/man/juno_frontend/#Progress-Meters-1). For Jupyter notebooks the default backend is [ConsoleProgressMonitor.jl](https://github.com/tkf/ConsoleProgressMonitor.jl). In all other cases progress logs are displayed with [TerminalLoggers.jl](https://github.com/c42f/TerminalLoggers.jl). Alternatively, if you provide a custom visualization backend, Turing uses it instead of the default backend. - diff --git a/_docs/using-turing/index.md b/_docs/using-turing/index.md deleted file mode 100644 index 30517aa51..000000000 --- a/_docs/using-turing/index.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -title: Turing Documentation ---- - - - - - -# Turing Documentation - - -Welcome to the documentation for Turing. - - - - - - -## Introduction - - -**Turing** is a general-purpose probabilistic programming language for robust, efficient Bayesian inference and decision making. Current features include: - - - * **General-purpose** probabilistic programming with an intuitive modelling interface; - * Robust, efficient [Hamiltonian Monte Carlo (HMC)](https://github.com/TuringLang/AdvancedHMC.jl) sampling for differentiable posterior distributions; - * **Particle MCMC** sampling for complex posterior distributions involving discrete variables and stochastic control flow; and - * Compositional inference via **Gibbs sampling** that combines particle MCMC, HMC and random-walk MH (RWMH). - diff --git a/_docs/using-turing/performancetips.md b/_docs/using-turing/performancetips.md deleted file mode 100644 index 3b1e905ed..000000000 --- a/_docs/using-turing/performancetips.md +++ /dev/null @@ -1,217 +0,0 @@ ---- -title: Performance Tips ---- - - - - - -# Performance Tips - - -This section briefly summarises a few common techniques to ensure good performance when using Turing. We refer to [the Julia documentation](https://docs.julialang.org/en/v1/manual/performance-tips/index.html) for general techniques to ensure good performance of Julia programs. - - - - - - -## Use multivariate distributions - - -It is generally preferable to use multivariate distributions if possible. - - -The following example: - - -```julia -@model function gmodel(x) - m ~ Normal() - for i = 1:length(x) - x[i] ~ Normal(m, 0.2) - end -end -``` - - -can be directly expressed more efficiently using a simple transformation: - - -```julia -using FillArrays - -@model function gmodel(x) - m ~ Normal() - x ~ MvNormal(Fill(m, length(x)), 0.2) -end -``` - - - - - - -## Choose your AD backend - - -Turing currently provides support for two different automatic differentiation (AD) backends. Generally, try to use `:forwarddiff` for models with few parameters and `:reversediff`, `:tracker` or `:zygote` for models with large parameter vectors or linear algebra operations. See [Automatic Differentiation](autodiff) for details. - - - - - - -## Special care for `:tracker` and `:zygote` - - -In case of `:tracker` and `:zygote`, it is necessary to avoid loops for now. This is mainly due to the reverse-mode AD backends `Tracker` and `Zygote` which are inefficient for such cases. `ReverseDiff` does better but vectorized operations will still perform better. - - -Avoiding loops can be done using `filldist(dist, N)` and `arraydist(dists)`. `filldist(dist, N)` creates a multivariate distribution that is composed of `N` identical and independent copies of the univariate distribution `dist` if `dist` is univariate, or it creates a matrix-variate distribution composed of `N` identical and idependent copies of the multivariate distribution `dist` if `dist` is multivariate. `filldist(dist, N, M)` can also be used to create a matrix-variate distribution from a univariate distribution `dist`. `arraydist(dists)` is similar to `filldist` but it takes an array of distributions `dists` as input. Writing a [custom distribution](advanced) with a custom adjoint is another option to avoid loops. - - - - - - -## Ensure that types in your model can be inferred - - -For efficient gradient-based inference, e.g. using HMC, NUTS or ADVI, it is important to ensure the types in your model can be inferred. - - -The following example with abstract types - - -```julia -@model function tmodel(x, y) - p,n = size(x) - params = Vector{Real}(undef, n) - for i = 1:n - params[i] ~ truncated(Normal(), 0, Inf) - end - - a = x * params - y ~ MvNormal(a, 1.0) -end -``` - - -can be transformed into the following representation with concrete types: - - -```julia -@model function tmodel(x, y, ::Type{T} = Float64) where {T} - p,n = size(x) - params = Vector{T}(undef, n) - for i = 1:n - params[i] ~ truncated(Normal(), 0, Inf) - end - - a = x * params - y ~ MvNormal(a, 1.0) -end -``` - - -Alternatively, you could use `filldist` in this example: - - -```julia -@model function tmodel(x, y) - params = filldist(truncated(Normal(), 0, Inf), size(x, 2)) - a = x * params - y ~ MvNormal(a, 1.0) -end -``` - - -Note that you can use `@code_warntype` to find types in your model definition that the compiler cannot infer. They are marked in red in the Julia REPL. - - -For example, consider the following simple program: - - -```julia -@model function tmodel(x) - p = Vector{Real}(undef, 1) - p[1] ~ Normal() - p = p .+ 1 - x ~ Normal(p[1]) -end -``` - - -We can use - - -```julia -using Random - -model = tmodel(1.0) - -@code_warntype model.f( - Random.GLOBAL_RNG, - model, - Turing.VarInfo(model), - Turing.SampleFromPrior(), - Turing.DefaultContext(), - model.args..., -) -``` - - -to inspect type inference in the model. - - - - - - -## Reuse Computations in Gibbs Sampling - - -Often when performing Gibbs sampling, one can save computational time by caching the output of expensive functions. The cached values can then be reused in future Gibbs sub-iterations which do not change the inputs to this expensive function. For example in the following model - - -```julia -@model function demo(x) - a ~ Gamma() - b ~ Normal() - c = function1(a) - d = function2(b) - x .~ Normal(c, d) -end -alg = Gibbs(MH(:a), MH(:b)) -sample(demo(zeros(10)), alg, 1000) -``` - - -when only updating `a` in a Gibbs sub-iteration, keeping `b` the same, the value of `d` doesn't change. And when only updating `b`, the value of `c` doesn't change. However, if `function1` and `function2` are expensive and are both run in every Gibbs sub-iteration, a lot of time would be spent computing values that we already computed before. Such a problem can be overcome using `Memoization.jl`. Memoizing a function lets us store and reuse the output of the function for every input it is called with. This has a slight time overhead but for expensive functions, the savings will be far greater. - - -To use `Memoization.jl`, simply define memoized versions of `function1` and `function2` as such: - - -```julia -using Memoization - -@memoize memoized_function1(args...) = function1(args...) -@memoize memoized_function2(args...) = function2(args...) -``` - - -Then define the `Turing` model using the new functions as such: - - -```julia -@model function demo(x) - a ~ Gamma() - b ~ Normal() - c = memoized_function1(a) - d = memoized_function2(b) - x .~ Normal(c, d) -end -``` - diff --git a/_docs/using-turing/quick-start.md b/_docs/using-turing/quick-start.md deleted file mode 100644 index 4db8fa716..000000000 --- a/_docs/using-turing/quick-start.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: Probablistic Programming in Thirty Seconds ---- - - - - - -# Probablistic Programming in Thirty Seconds - - -If you are already well-versed in probabalistic programming and just want to take a quick look at how Turing's syntax works or otherwise just want a model to start with, we have provided a Bayesian coin-flipping model to play with. - - -This example can be run on however you have Julia installed (see [Getting Started]({{site.baseurl}}/docs/using-turing/get-started)), but you will need to install the packages `Turing` and `StatsPlots` if you have not done so already. - - -This is an excerpt from a more formal example introducing probabalistic programming which can be found in Jupyter notebook form [here](https://nbviewer.jupyter.org/github/TuringLang/TuringTutorials/blob/master/0_Introduction.ipynb) or as part of the documentation website [here]({{site.baseurl}}/tutorials). - - -```julia -# Import libraries. -using Turing, StatsPlots, Random - -# Set the true probability of heads in a coin. -p_true = 0.5 - -# Iterate from having seen 0 observations to 100 observations. -Ns = 0:100 - -# Draw data from a Bernoulli distribution, i.e. draw heads or tails. -Random.seed!(12) -data = rand(Bernoulli(p_true), last(Ns)) - -# Declare our Turing model. -@model function coinflip(y) - # Our prior belief about the probability of heads in a coin. - p ~ Beta(1, 1) - - # The number of observations. - N = length(y) - for n in 1:N - # Heads or tails of a coin are drawn from a Bernoulli distribution. - y[n] ~ Bernoulli(p) - end -end - -# Settings of the Hamiltonian Monte Carlo (HMC) sampler. -iterations = 1000 -ϵ = 0.05 -τ = 10 - -# Start sampling. -chain = sample(coinflip(data), HMC(ϵ, τ), iterations) - -# Plot a summary of the sampling process for the parameter p, i.e. the probability of heads in a coin. -histogram(chain[:p]) -``` - diff --git a/_docs/using-turing/sampler-figs/samplers-1.svg b/_docs/using-turing/sampler-figs/samplers-1.svg deleted file mode 100644 index c32b05a16..000000000 --- a/_docs/using-turing/sampler-figs/samplers-1.svg +++ /dev/null @@ -1,2725 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 - - -2 - - -3 - - -4 - - --5 - - -0 - - -5 - - -10 - - --300 - - --200 - - --100 - - -0 - - -Gibbs{HMC, PG} - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-figs/samplers-2.svg b/_docs/using-turing/sampler-figs/samplers-2.svg deleted file mode 100644 index 77b377629..000000000 --- a/_docs/using-turing/sampler-figs/samplers-2.svg +++ /dev/null @@ -1,2479 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -4.0 - - -4.5 - - --6.5 - - --6.0 - - --5.5 - - --5.0 - - --4.5 - - --4.0 - - --3.5 - - --80 - - --70 - - --60 - - --50 - - --40 - - --30 - - -HMC - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-figs/samplers-3.svg b/_docs/using-turing/sampler-figs/samplers-3.svg deleted file mode 100644 index 76db8c4dc..000000000 --- a/_docs/using-turing/sampler-figs/samplers-3.svg +++ /dev/null @@ -1,2516 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1.5 - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -4.0 - - -4.5 - - --1 - - -0 - - -1 - - -2 - - -3 - - --125 - - --100 - - --75 - - --50 - - --25 - - -HMCDA - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-figs/samplers-4.svg b/_docs/using-turing/sampler-figs/samplers-4.svg deleted file mode 100644 index 08ecc3f65..000000000 --- a/_docs/using-turing/sampler-figs/samplers-4.svg +++ /dev/null @@ -1,2635 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.5 - - -1.0 - - -1.5 - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - --1 - - -0 - - -1 - - -2 - - -3 - - --175 - - --150 - - --125 - - --100 - - --75 - - --50 - - --25 - - -MH() - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-figs/samplers-5.svg b/_docs/using-turing/sampler-figs/samplers-5.svg deleted file mode 100644 index c656086a6..000000000 --- a/_docs/using-turing/sampler-figs/samplers-5.svg +++ /dev/null @@ -1,2426 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1.5 - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -4.0 - - -4.5 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - --80 - - --70 - - --60 - - --50 - - --40 - - --30 - - --20 - - -NUTS(0.65) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-figs/samplers-6.svg b/_docs/using-turing/sampler-figs/samplers-6.svg deleted file mode 100644 index 76badce95..000000000 --- a/_docs/using-turing/sampler-figs/samplers-6.svg +++ /dev/null @@ -1,2222 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -2 - - -3 - - -4 - - -5 - - --7 - - --6 - - --5 - - --4 - - --80 - - --70 - - --60 - - --50 - - --40 - - --30 - - -NUTS(0.95) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-figs/samplers-7.svg b/_docs/using-turing/sampler-figs/samplers-7.svg deleted file mode 100644 index c0dc125bc..000000000 --- a/_docs/using-turing/sampler-figs/samplers-7.svg +++ /dev/null @@ -1,2562 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -4.0 - - -4.5 - - -0 - - -2 - - -4 - - -6 - - --70 - - --60 - - --50 - - --40 - - --30 - - --20 - - -NUTS(0.2) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-figs/samplers-8.svg b/_docs/using-turing/sampler-figs/samplers-8.svg deleted file mode 100644 index 4382ee70e..000000000 --- a/_docs/using-turing/sampler-figs/samplers-8.svg +++ /dev/null @@ -1,2691 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 - - -2 - - -3 - - -4 - - --5 - - -0 - - -5 - - -10 - - --400 - - --300 - - --200 - - --100 - - -0 - - -PG(20) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-figs/samplers-9.svg b/_docs/using-turing/sampler-figs/samplers-9.svg deleted file mode 100644 index baefbb519..000000000 --- a/_docs/using-turing/sampler-figs/samplers-9.svg +++ /dev/null @@ -1,2616 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -1 - - -2 - - -3 - - -4 - - --5 - - -0 - - -5 - - -10 - - --800 - - --600 - - --400 - - --200 - - -0 - - -PG(50) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_docs/using-turing/sampler-viz.md b/_docs/using-turing/sampler-viz.md deleted file mode 100644 index 4ad49681f..000000000 --- a/_docs/using-turing/sampler-viz.md +++ /dev/null @@ -1,290 +0,0 @@ ---- -title: Sampler Visualization -mathjax: true ---- - - - - - -# Sampler Visualization - - - - - - -## Introduction - - - - - - -## The Code - - -For each sampler, we will use the same code to plot sampler paths. The block below loads the relevant libraries and defines a function for plotting the sampler's trajectory across the posterior. - - -The Turing model definition used here is not especially practical, but it is designed in such a way as to produce visually interesting posterior surfaces to show how different samplers move along the distribution. - - -```julia -ENV["GKS_ENCODING"] = "utf-8" # Allows the use of unicode characters in Plots.jl -using Plots -using StatsPlots -using Turing -using Bijectors -using Random -using DynamicPPL: getlogp, settrans!, getval, reconstruct, vectorize, setval! - -# Set a seed. -Random.seed!(0) - -# Define a strange model. -@model gdemo(x) = begin - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - bumps = sin(m) + cos(m) - m = m + 5*bumps - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end - return s, m -end - -# Define our data points. -x = [1.5, 2.0, 13.0, 2.1, 0.0] - -# Set up the model call, sample from the prior. -model = gdemo(x) -vi = Turing.VarInfo(model) - -# Convert the variance parameter to the real line before sampling. -# Note: We only have to do this here because we are being very hands-on. -# Turing will handle all of this for you during normal sampling. -dist = InverseGamma(2,3) -svn = vi.metadata.s.vns[1] -mvn = vi.metadata.m.vns[1] -setval!(vi, vectorize(dist, Bijectors.link(dist, reconstruct(dist, getval(vi, svn)))), svn) -settrans!(vi, true, svn) - -# Evaluate surface at coordinates. -function evaluate(m1, m2) - spl = Turing.SampleFromPrior() - vi[svn] = [m1] - vi[mvn] = [m2] - model(vi, spl) - getlogp(vi) -end - -function plot_sampler(chain; label="") - # Extract values from chain. - val = get(chain, [:s, :m, :lp]) - ss = link.(Ref(InverseGamma(2, 3)), val.s) - ms = val.m - lps = val.lp - - # How many surface points to sample. - granularity = 100 - - # Range start/stop points. - spread = 0.5 - σ_start = minimum(ss) - spread * std(ss); σ_stop = maximum(ss) + spread * std(ss); - μ_start = minimum(ms) - spread * std(ms); μ_stop = maximum(ms) + spread * std(ms); - σ_rng = collect(range(σ_start, stop=σ_stop, length=granularity)) - μ_rng = collect(range(μ_start, stop=μ_stop, length=granularity)) - - # Make surface plot. - p = surface(σ_rng, μ_rng, evaluate, - camera=(30, 65), - # ticks=nothing, - colorbar=false, - color=:inferno, - title=label) - - line_range = 1:length(ms) - - scatter3d!(ss[line_range], ms[line_range], lps[line_range], - mc =:viridis, marker_z=collect(line_range), msw=0, - legend=false, colorbar=false, alpha=0.5, - xlabel="σ", ylabel="μ", zlabel="Log probability", - title=label) - - return p -end; -``` - - - - - - -## Samplers - - - - - - -### Gibbs - - -Gibbs sampling tends to exhibit a "jittery" trajectory. The example below combines `HMC` and `PG` sampling to traverse the posterior. - - -```julia -c = sample(model, Gibbs(HMC(0.01, 5, :s), PG(20, :m)), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-1.svg) - - - - - - -### HMC - - -Hamiltonian Monte Carlo (HMC) sampling is a typical sampler to use, as it tends to be fairly good at converging in a efficient manner. It can often be tricky to set the correct parameters for this sampler however, and the `NUTS` sampler is often easier to run if you don't want to spend too much time fiddling with step size and and the number of steps to take. Note however that `HMC` does not explore the positive values μ very well, likely due to the leapfrop and step size parameter settings. - - -```julia -c = sample(model, HMC(0.01, 10), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-2.svg) - - - - - - -### HMCDA - - -The HMCDA sampler is an implementation of the Hamiltonian Monte Carlo with Dual Averaging algorithm found in the paper "The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo" by Hoffman and Gelman (2011). The paper can be found on [arXiv](https://arxiv.org/abs/1111.4246) for the interested reader. - - -```julia -c = sample(model, HMCDA(200, 0.65, 0.3), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-3.svg) - - - - - - -### MH - - -Metropolis-Hastings (MH) sampling is one of the earliest Markov Chain Monte Carlo methods. MH sampling does not "move" a lot, unlike many of the other samplers implemented in Turing. Typically a much longer chain is required to converge to an appropriate parameter estimate. - - -The plot below only uses 1,000 iterations of Metropolis-Hastings. - - -```julia -c = sample(model, MH(), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-4.svg) - - -As you can see, the MH sampler doesn't move parameter estimates very often. - - - - - - -### NUTS - - -The No U-Turn Sampler (NUTS) is an implementation of the algorithm found in the paper "The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo" by Hoffman and Gelman (2011). The paper can be found on [arXiv](https://arxiv.org/abs/1111.4246) for the interested reader. - - -NUTS tends to be very good at traversing complex posteriors quickly. - - -```julia -c = sample(model, NUTS(0.65), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-5.svg) - - -The only parameter that needs to be set other than the number of iterations to run is the target acceptance rate. In the Hoffman and Gelman paper, they note that a target acceptance rate of 0.65 is typical. - - -Here is a plot showing a very high acceptance rate. Note that it appears to "stick" to a mode and is not particularly good at exploring the posterior as compared to the 0.65 target acceptance ratio case. - - -```julia -c = sample(model, NUTS(0.95), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-6.svg) - - -An exceptionally low acceptance rate will show very few moves on the posterior: - - -```julia -c = sample(model, NUTS(0.2), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-7.svg) - - - - - - -### PG - - -The Particle Gibbs (PG) sampler is an implementation of an algorithm from the paper "Particle Markov chain Monte Carlo methods" by Andrieu, Doucet, and Holenstein (2010). The interested reader can learn more [here](https://rss.onlinelibrary.wiley.com/doi/full/10.1111/j.1467-9868.2009.00736.x). - - -The two parameters are the number of particles, and the number of iterations. The plot below shows the use of 20 particles. - - -```julia -c = sample(model, PG(20), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-8.svg) - - -Next, we plot using 50 particles. - - -```julia -c = sample(model, PG(50), 1000) -plot_sampler(c) -``` - - -![](sampler-figs/samplers-9.svg) - diff --git a/_packages/packages.yml b/_packages/packages.yml deleted file mode 100644 index b10687bc1..000000000 --- a/_packages/packages.yml +++ /dev/null @@ -1,2 +0,0 @@ -- package: AdvancedHMC - baseurl: advancedhmc \ No newline at end of file diff --git a/_posts/.POSTING_GUIDE.MD b/_posts/.POSTING_GUIDE.MD deleted file mode 100644 index 481a45c98..000000000 --- a/_posts/.POSTING_GUIDE.MD +++ /dev/null @@ -1,12 +0,0 @@ -# How to write a blog post. - -1. Place all blog posts as markdown files in `docs/_posts`. -2. Make sure you use the naming convention "YYYY-MM-DD-some-short-title.md". -3. Include a YAML header in the very first line with the flags `title` and `author`, here's an example: - -```yaml ---- -title: Turing's Blog -author: Cameron Pfiffer ---- -``` diff --git a/_posts/2020-09-11-gsoc.md b/_posts/2020-09-11-gsoc.md index 17ec0eea1..458ae47bd 100644 --- a/_posts/2020-09-11-gsoc.md +++ b/_posts/2020-09-11-gsoc.md @@ -1,5 +1,5 @@ --- -title: "Google Summer of Code Outcomes" +title: "Google Summer of Code 2020" author: Cameron Pfiffer date: 2020-09-11 --- diff --git a/_tutorials/0_Introduction.md b/_tutorials/0_Introduction.md deleted file mode 100644 index 1032e8ec0..000000000 --- a/_tutorials/0_Introduction.md +++ /dev/null @@ -1,224 +0,0 @@ ---- -title: Introduction to Turing -permalink: /:collection/:name/ ---- - -# Introduction to Turing - -## Introduction - -This is the first of a series of tutorials on the universal probabilistic programming language **Turing**. - -Turing is a probabilistic programming system written entirely in Julia. It has an intuitive modelling syntax and supports a wide range of sampling-based inference algorithms. Most importantly, Turing inference is composable: it combines Markov chain sampling operations on subsets of model variables, e.g. using a combination of a Hamiltonian Monte Carlo (HMC) engine and a particle Gibbs (PG) engine. This composable inference engine allows the user to easily switch between black-box style inference methods such as HMC and customized inference methods. - -Familiarity with Julia is assumed through out this tutorial. If you are new to Julia, [Learning Julia](https://julialang.org/learning/) is a good starting point. - -For users new to Bayesian machine learning, please consider more thorough introductions to the field, such as [Pattern Recognition and Machine Learning](https://www.springer.com/us/book/9780387310732). This tutorial tries to provide an intuition for Bayesian inference and gives a simple example on how to use Turing. Note that this is not a comprehensive introduction to Bayesian machine learning. - -### Coin Flipping Without Turing -The following example illustrates the effect of updating our beliefs with every piece of new evidence we observe. In particular, assume that we are unsure about the probability of heads in a coin flip. To get an intuitive understanding of what "updating our beliefs" is, we will visualize the probability of heads in a coin flip after each observed evidence. - -First, let's load some of the packages we need to flip a coin (`Random`, `Distributions`) and show our results (`Plots`). You will note that Turing is not an import here — we do not need it for this example. If you are already familiar with posterior updates, you can proceed to the next step. - - -```julia -# Using Base modules. -using Random - -# Load a plotting library. -using Plots - -# Load the distributions library. -using Distributions -``` - -Next, we configure our posterior update model. First, let's set the true probability that any coin flip will turn up heads and set the number of coin flips we will show our model: - - -```julia -# Set the true probability of heads in a coin. -p_true = 0.5 - -# Iterate from having seen 0 observations to 100 observations. -Ns = 0:100; -``` - -We will now use the Bernoulli distribution to flip 100 coins, and collect the results in a variable called `data`: - - -```julia -# Draw data from a Bernoulli distribution, i.e. draw heads or tails. -Random.seed!(12) -data = rand(Bernoulli(p_true), last(Ns)) - -# Here's what the first five coin flips look like: -data[1:5] -``` - - - - - 5-element Array{Bool,1}: - 1 - 0 - 1 - 1 - 0 - - - -After flipping all our coins, we want to set a prior belief about what we think the distribution of coin flips look like. In this case, we are going to choose a common prior distribution called the [Beta](https://en.wikipedia.org/wiki/Beta_distribution) distribution. - - -```julia -# Our prior belief about the probability of heads in a coin toss. -prior_belief = Beta(1, 1); -``` - -With our priors set and our data at hand, we can perform Bayesian inference. - -This is a fairly simple process. We expose one additional coin flip to our model every iteration, such that the first run only sees the first coin flip, while the last iteration sees all the coin flips. Then, we set the `updated_belief` variable to an updated version of the original Beta distribution that accounts for the new proportion of heads and tails. - -For the mathematically inclined, the `Beta` distribution is updated by adding each coin flip to the distribution's $$\alpha$$ and $$\beta$$ parameters, which are initially defined as $$\alpha = 1, \beta = 1$$. Over time, with more and more coin flips, $$\alpha$$ and $$\beta$$ will be approximately equal to each other as we are equally likely to flip a heads or a tails, and the plot of the beta distribution will become more tightly centered around 0.5. - -This works because mean of the `Beta` distribution is defined as the following: - -\$\$ \text{E}[\text{Beta}] = \dfrac{\alpha}{\alpha+\beta} \$\$ - -Which is 0.5 when $$\alpha = \beta$$, as we expect for a large enough number of coin flips. As we increase the number of samples, our variance will also decrease, such that the distribution will reflect less uncertainty about the probability of receiving a heads. The definition of the variance for the `Beta` distribution is the following: - -\$\$ \text{var}[\text{Beta}] = \dfrac{\alpha\beta}{(\alpha + \beta)^2 (\alpha + \beta + 1)} \$\$ - -The intuition about this definition is that the variance of the distribution will approach 0 with more and more samples, as the denominator will grow faster than will the numerator. More samples means less variance. - - -```julia -# Import StatsPlots for animating purposes. -using StatsPlots - -# Make an animation. -animation = @gif for (i, N) in enumerate(Ns) - - # Count the number of heads and tails. - heads = sum(data[1:i-1]) - tails = N - heads - - # Update our prior belief in closed form (this is possible because we use a conjugate prior). - updated_belief = Beta(prior_belief.α + heads, prior_belief.β + tails) - - # Plotting - plot(updated_belief, - size = (500, 250), - title = "Updated belief after $$N observations", - xlabel = "probability of heads", - ylabel = "", - legend = nothing, - xlim = (0,1), - fill=0, α=0.3, w=3) - vline!([p_true]) -end; -``` - - ┌ Info: Saved animation to - │ fn = /home/cameron/code/TuringTutorials/tmp.gif - └ @ Plots /home/cameron/.julia/packages/Plots/Xnzc7/src/animation.jl:104 - - -![animation](https://user-images.githubusercontent.com/7974003/44995702-37c1b200-af9c-11e8-8b26-c88a528956af.gif) - -The animation above shows that with increasing evidence our belief about the probability of heads in a coin flip slowly adjusts towards the true value. The orange line in the animation represents the true probability of seeing heads on a single coin flip, while the mode of the distribution shows what the model believes the probability of a heads is given the evidence it has seen. - -### Coin Flipping With Turing - -In the previous example, we used the fact that our prior distribution is a [conjugate prior](https://en.wikipedia.org/wiki/Conjugate_prior). Note that a closed-form expression (the `updated_belief` expression) for the posterior is not accessible in general and usually does not exist for more interesting models. - -We are now going to move away from the closed-form expression above and specify the same model using **Turing**. To do so, we will first need to import `Turing`, `MCMCChains`, `Distributions`, and `StatPlots`. `MCMCChains` is a library built by the Turing team to help summarize Markov Chain Monte Carlo (MCMC) simulations, as well as a variety of utility functions for diagnostics and visualizations. - - -```julia -# Load Turing and MCMCChains. -using Turing, MCMCChains - -# Load the distributions library. -using Distributions - -# Load StatsPlots for density plots. -using StatsPlots -``` - -First, we define the coin-flip model using Turing. - - -```julia -@model coinflip(y) = begin - - # Our prior belief about the probability of heads in a coin. - p ~ Beta(1, 1) - - # The number of observations. - N = length(y) - for n in 1:N - # Heads or tails of a coin are drawn from a Bernoulli distribution. - y[n] ~ Bernoulli(p) - end -end; -``` - -After defining the model, we can approximate the posterior distribution by drawing samples from the distribution. In this example, we use a [Hamiltonian Monte Carlo](https://en.wikipedia.org/wiki/Hamiltonian_Monte_Carlo) sampler to draw these samples. Later tutorials will give more information on the samplers available in Turing and discuss their use for different models. - - -```julia -# Settings of the Hamiltonian Monte Carlo (HMC) sampler. -iterations = 1000 -ϵ = 0.05 -τ = 10 - -# Start sampling. -chain = sample(coinflip(data), HMC(ϵ, τ), iterations, progress=false); -``` - -After finishing the sampling process, we can visualize the posterior distribution approximated using Turing against the posterior distribution in closed-form. We can extract the chain data from the sampler using the `Chains(chain[:p])` function, exported from the `MCMCChain` module. `Chains(chain[:p])` creates an instance of the `Chain` type which summarizes the MCMC simulation — the `MCMCChain` module supports numerous tools for plotting, summarizing, and describing variables of type `Chain`. - - -```julia -# Construct summary of the sampling process for the parameter p, i.e. the probability of heads in a coin. -p_summary = chain[:p] -plot(p_summary, seriestype = :histogram) -``` - - - - -![svg](../0_Introduction_files/0_Introduction_21_0.svg) - - - -Now we can build our plot: - - -```julia -# Compute the posterior distribution in closed-form. -N = length(data) -heads = sum(data) -updated_belief = Beta(prior_belief.α + heads, prior_belief.β + N - heads) - -# Visualize a blue density plot of the approximate posterior distribution using HMC (see Chain 1 in the legend). -p = plot(p_summary, seriestype = :density, xlim = (0,1), legend = :best, w = 2, c = :blue) - -# Visualize a green density plot of posterior distribution in closed-form. -plot!(p, range(0, stop = 1, length = 100), pdf.(Ref(updated_belief), range(0, stop = 1, length = 100)), - xlabel = "probability of heads", ylabel = "", title = "", xlim = (0,1), label = "Closed-form", - fill=0, α=0.3, w=3, c = :lightgreen) - -# Visualize the true probability of heads in red. -vline!(p, [p_true], label = "True probability", c = :red) -``` - - - - -![svg](../0_Introduction_files/0_Introduction_23_0.svg) - - - -As we can see, the Turing model closely approximates the true probability. Hopefully this tutorial has provided an easy-to-follow, yet informative introduction to Turing's simpler applications. More advanced usage will be demonstrated in later tutorials. diff --git a/_tutorials/0_Introduction_files/0_Introduction_21_0.svg b/_tutorials/0_Introduction_files/0_Introduction_21_0.svg deleted file mode 100644 index e6e2c2f58..000000000 --- a/_tutorials/0_Introduction_files/0_Introduction_21_0.svg +++ /dev/null @@ -1,216 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/0_Introduction_files/0_Introduction_23_0.svg b/_tutorials/0_Introduction_files/0_Introduction_23_0.svg deleted file mode 100644 index 0405601bb..000000000 --- a/_tutorials/0_Introduction_files/0_Introduction_23_0.svg +++ /dev/null @@ -1,176 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_BayesianDiffEq.md b/_tutorials/10_BayesianDiffEq.md deleted file mode 100644 index 2f1a8b018..000000000 --- a/_tutorials/10_BayesianDiffEq.md +++ /dev/null @@ -1,660 +0,0 @@ ---- -title: Bayesian Estimation of Differential Equations -permalink: /:collection/:name/ ---- - -# Bayesian Estimation of Differential Equations - -Most of the scientific community deals with the basic problem of trying to mathematically model the reality around them and this often involves dynamical systems. The general trend to model these complex dynamical systems is through the use of differential equations. Differential equation models often have non-measurable parameters. The popular “forward-problem” of simulation consists of solving the differential equations for a given set of parameters, the “inverse problem” to simulation, known as parameter estimation, is the process of utilizing data to determine these model parameters. Bayesian inference provides a robust approach to parameter estimation with quantified uncertainty. - - -```julia -using Turing, Distributions, DataFrames, DifferentialEquations, DiffEqSensitivity - -# Import MCMCChain, Plots, and StatsPlots for visualizations and diagnostics. -using MCMCChains, Plots, StatsPlots - -# Set a seed for reproducibility. -using Random -Random.seed!(12); -``` - -## The Lotka-Volterra Model - -The Lotka–Volterra equations, also known as the predator–prey equations, are a pair of first-order nonlinear differential equations, frequently used to describe the dynamics of biological systems in which two species interact, one as a predator and the other as prey. The populations change through time according to the pair of equations: - -$$\frac{dx}{dt} = (\alpha - \beta y)x$$ - -$$\frac{dy}{dt} = (\delta x - \gamma)y$$ - - - -```julia -function lotka_volterra(du,u,p,t) - x, y = u - α, β, δ, γ = p - du[1] = dx = (α - β*y)x - du[2] = dy = (δ*x - γ)y -end -p = [1.5, 1.0, 3.0, 1.0] -u0 = [1.0,1.0] -prob = ODEProblem(lotka_volterra,u0,(0.0,10.0),p) -sol = solve(prob,Tsit5()) -plot(sol) -``` - - - - -![svg](../10_BayesianDiffEq_files/10_BayesianDiffEq_3_0.svg) - - - -We'll generate the data to use for the parameter estimation from simulation. -With the `saveat` [argument](https://docs.sciml.ai/latest/basics/common_solver_opts/) we specify that the solution is stored only at `0.1` time units. - - -```julia -odedata = Array(solve(prob,Tsit5(),saveat=0.1)) -``` - - - - - 2×101 Array{Float64,2}: - 1.0 1.03981 1.05332 1.03247 0.972908 … 0.133965 0.148601 0.165247 - 1.0 1.22939 1.52387 1.88714 2.30908 0.476902 0.450153 0.426924 - - - -## Fitting Lotka-Volterra with DiffEqBayes - -[DiffEqBayes.jl](https://github.com/SciML/DiffEqBayes.jl) is a high level package that set of extension functionality for estimating the parameters of differential equations using Bayesian methods. It allows the choice of using CmdStan.jl, Turing.jl, DynamicHMC.jl and ApproxBayes.jl to perform a Bayesian estimation of a differential equation problem specified via the DifferentialEquations.jl interface. You can read the [docs](https://docs.sciml.ai/latest/analysis/parameter_estimation/#Bayesian-Methods-1) for an understanding of the available functionality. - - -```julia -using DiffEqBayes -t = 0:0.1:10.0 -priors = [truncated(Normal(1.5,0.5),0.5,2.5),truncated(Normal(1.2,0.5),0,2),truncated(Normal(3.0,0.5),1,4),truncated(Normal(1.0,0.5),0,2)] -bayesian_result_turing = turing_inference(prob,Tsit5(),t,odedata,priors,num_samples=10_000) -``` - - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Info: Found initial step size - │ ϵ = 0.00625 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/GMBTf/src/inference/hmc.jl:629 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - - - - - - Object of type Chains, with data of type 9000×17×1 Array{Float64,3} - - Iterations = 1:9000 - Thinning interval = 1 - Chains = 1 - Samples per chain = 9000 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = theta[1], theta[2], theta[3], theta[4], σ[1] - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ───────── ────── - theta[1] 2.3263 0.1073 0.0011 0.0021 2202.3643 1.0000 - theta[2] 1.5434 0.0957 0.0010 0.0019 2575.4033 1.0002 - theta[3] 3.1259 0.1983 0.0021 0.0031 4127.1344 1.0000 - theta[4] 1.8356 0.0827 0.0009 0.0017 2189.2825 1.0000 - σ[1] 0.8569 0.0436 0.0005 0.0005 6856.5421 0.9999 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - theta[1] 2.1185 2.2428 2.3337 2.4169 2.4916 - theta[2] 1.3655 1.4750 1.5422 1.6075 1.7367 - theta[3] 2.7571 2.9893 3.1166 3.2546 3.5440 - theta[4] 1.6902 1.7708 1.8307 1.9006 1.9868 - σ[1] 0.7755 0.8266 0.8551 0.8847 0.9484 - - - - -The estimated parameters are clearly very close to the desired parameter values. We can also check that the chains have converged in the plot. - - -```julia -plot(bayesian_result_turing) -``` - - - - -![svg](../10_BayesianDiffEq_files/10_BayesianDiffEq_9_0.svg) - - - -## Direct Handling of Bayesian Estimation with Turing - -You could want to do some sort of reduction with the differential equation's solution or use it in some other way as well. In those cases DiffEqBayes might not be useful. Turing and DifferentialEquations are completely composable and you can write of the differential equation inside a Turing `@model` and it will just work. - -We can rewrite the Lotka Volterra parameter estimation problem with a Turing `@model` interface as below - - -```julia -Turing.setadbackend(:forwarddiff) - -@model function fitlv(data) - σ ~ InverseGamma(2, 3) - α ~ truncated(Normal(1.5,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.5),0,2) - γ ~ truncated(Normal(3.0,0.5),1,4) - δ ~ truncated(Normal(1.0,0.5),0,2) - - p = [α,β,γ,δ] - prob = ODEProblem(lotka_volterra,u0,(0.0,10.0),p) - predicted = solve(prob,Tsit5(),saveat=0.1) - - for i = 1:length(predicted) - data[:,i] ~ MvNormal(predicted[i], σ) - end -end - -model = fitlv(odedata) -chain = sample(model, NUTS(.65),10000) -``` - - ┌ Info: Found initial step size - │ ϵ = 0.2 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/GMBTf/src/inference/hmc.jl:629 - Sampling: 100%|█████████████████████████████████████████| Time: 0:02:48 - - - - - - Object of type Chains, with data of type 9000×17×1 Array{Float64,3} - - Iterations = 1:9000 - Thinning interval = 1 - Chains = 1 - Samples per chain = 9000 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ───────── ────── - α 1.4999 0.0060 0.0001 0.0001 2341.1779 0.9999 - β 0.9999 0.0037 0.0000 0.0001 2440.6968 0.9999 - γ 3.0001 0.0047 0.0000 0.0001 4070.6419 1.0003 - δ 1.0001 0.0032 0.0000 0.0001 2324.4733 0.9999 - σ 0.0151 0.0011 0.0000 0.0000 4591.2728 0.9999 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 1.4881 1.4960 1.4998 1.5038 1.5118 - β 0.9925 0.9975 0.9999 1.0024 1.0074 - γ 2.9911 2.9970 3.0000 3.0032 3.0095 - δ 0.9937 0.9979 1.0001 1.0022 1.0066 - σ 0.0131 0.0143 0.0150 0.0158 0.0173 - - - - -## Scaling to Large Models: Adjoint Sensitivities - -DifferentialEquations.jl's efficiency for large stiff models has been shown in multiple [benchmarks](https://github.com/SciML/DiffEqBenchmarks.jl). To learn more about how to optimize solving performance for stiff problems you can take a look at the [docs](https://docs.sciml.ai/latest/tutorials/advanced_ode_example/). - -[Sensitivity analysis](https://docs.sciml.ai/latest/analysis/sensitivity/), or automatic differentiation (AD) of the solver, is provided by the DiffEq suite. The model sensitivities are the derivatives of the solution $$u(t)$$ with respect to the parameters. Specifically, the local sensitivity of the solution to a parameter is defined by how much the solution would change by changes in the parameter. Sensitivity analysis provides a cheap way to calculate the gradient of the solution which can be used in parameter estimation and other optimization tasks. - - -The AD ecosystem in Julia allows you to switch between forward mode, reverse mode, source to source and other choices of AD and have it work with any Julia code. For a user to make use of this within [SciML](https://sciml.ai), [high level interactions in `solve`](https://docs.sciml.ai/latest/analysis/sensitivity/#High-Level-Interface:-sensealg-1) automatically plug into those AD systems to allow for choosing advanced sensitivity analysis (derivative calculation) [methods](https://docs.sciml.ai/latest/analysis/sensitivity/#Sensitivity-Algorithms-1). - -More theoretical details on these methods can be found at: https://docs.sciml.ai/latest/extras/sensitivity_math/. - -While these sensitivity analysis methods may seem complicated (and they are!), using them is dead simple. Here is a version of the Lotka-Volterra model with adjoints enabled. - -All we had to do is switch the AD backend to one of the adjoint-compatible backends (ReverseDiff, Tracker, or Zygote) and boom the system takes over and we're using adjoint methods! Notice that on this model adjoints are slower. This is because adjoints have a higher overhead on small parameter models and we suggest only using these methods for models with around 100 parameters or more. For more details, see https://arxiv.org/abs/1812.01892. - - -```julia -Turing.setadbackend(:zygote) -@model function fitlv(data) - σ ~ InverseGamma(2, 3) - α ~ truncated(Normal(1.5,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.5),0,2) - γ ~ truncated(Normal(3.0,0.5),1,4) - δ ~ truncated(Normal(1.0,0.5),0,2) - p = [α,β,γ,δ] - prob = ODEProblem(lotka_volterra,u0,(0.0,10.0),p) - predicted = solve(prob,saveat=0.1) - for i = 1:length(predicted) - data[:,i] ~ MvNormal(predicted[i], σ) - end -end; -model = fitlv(odedata) -chain = sample(model, NUTS(.65),1000) -``` - - ┌ Info: Found initial step size - │ ϵ = 0.2 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/GMBTf/src/inference/hmc.jl:629 - Sampling: 100%|█████████████████████████████████████████| Time: 0:10:42 - - - - - - Object of type Chains, with data of type 500×17×1 Array{Float64,3} - - Iterations = 1:500 - Thinning interval = 1 - Chains = 1 - Samples per chain = 500 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ──────── ────── - α 1.4997 0.0052 0.0002 0.0003 201.5277 1.0046 - β 0.9999 0.0033 0.0001 0.0001 219.1974 1.0027 - γ 3.0003 0.0047 0.0002 0.0003 290.3332 1.0014 - δ 1.0002 0.0029 0.0001 0.0002 210.0807 1.0046 - σ 0.0151 0.0010 0.0000 0.0001 246.6502 1.0017 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 1.4892 1.4962 1.5002 1.5030 1.5108 - β 0.9934 0.9978 1.0000 1.0019 1.0066 - γ 2.9910 2.9971 3.0002 3.0039 3.0084 - δ 0.9943 0.9983 1.0000 1.0021 1.0060 - σ 0.0131 0.0143 0.0151 0.0158 0.0172 - - - - -Now we can exercise control of the sensitivity analysis method that is used by using the `sensealg` keyword argument. Let's choose the `InterpolatingAdjoint` from the available AD [methods](https://docs.sciml.ai/latest/analysis/sensitivity/#Sensitivity-Algorithms-1) and enable a compiled ReverseDiff vector-Jacobian product: - - -```julia -@model function fitlv(data) - σ ~ InverseGamma(2, 3) - α ~ truncated(Normal(1.5,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.5),0,2) - γ ~ truncated(Normal(3.0,0.5),1,4) - δ ~ truncated(Normal(1.0,0.5),0,2) - p = [α,β,γ,δ] - prob = ODEProblem(lotka_volterra,u0,(0.0,10.0),p) - predicted = solve(prob,saveat=0.1,sensealg=InterpolatingAdjoint(autojacvec=ReverseDiffVJP(true))) - for i = 1:length(predicted) - data[:,i] ~ MvNormal(predicted[i], σ) - end -end; -model = fitlv(odedata) -@time chain = sample(model, NUTS(.65),1000) -``` - - ┌ Info: Found initial step size - │ ϵ = 0.2 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/GMBTf/src/inference/hmc.jl:629 - Sampling: 11%|████▍ | ETA: 0:06:27┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 13%|█████▍ | ETA: 0:05:58┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 15%|██████▎ | ETA: 0:05:27┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 21%|████████▌ | ETA: 0:04:20┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 23%|█████████▎ | ETA: 0:04:03┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 24%|██████████ | ETA: 0:03:48┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 28%|███████████▌ | ETA: 0:03:27┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 29%|███████████▊ | ETA: 0:03:24┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 29%|████████████ | ETA: 0:03:20┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 36%|███████████████ | ETA: 0:02:45┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 37%|███████████████▏ | ETA: 0:02:44┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 39%|████████████████ | ETA: 0:02:36┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 46%|██████████████████▉ | ETA: 0:02:08┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 48%|███████████████████▊ | ETA: 0:02:03┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 49%|████████████████████▏ | ETA: 0:02:01┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 50%|████████████████████▎ | ETA: 0:02:00┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 100%|█████████████████████████████████████████| Time: 0:03:32 - - - 225.663919 seconds (1.41 G allocations: 66.216 GiB, 5.25% gc time) - - - - - - Object of type Chains, with data of type 500×17×1 Array{Float64,3} - - Iterations = 1:500 - Thinning interval = 1 - Chains = 1 - Samples per chain = 500 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ──────── ────── - α 0.9122 0.2810 0.0126 0.0152 211.4497 0.9992 - β 1.8499 0.1141 0.0051 0.0055 302.7650 1.0018 - γ 2.5879 0.3299 0.0148 0.0228 307.5110 0.9997 - δ 0.1259 0.0221 0.0010 0.0007 219.5371 1.0006 - σ 0.8746 0.0437 0.0020 0.0017 342.6660 1.0008 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 0.5060 0.6920 0.8932 1.0874 1.5467 - β 1.5810 1.7796 1.8709 1.9437 1.9873 - γ 1.9519 2.3707 2.5999 2.8158 3.1966 - δ 0.0843 0.1103 0.1245 0.1410 0.1704 - σ 0.7984 0.8444 0.8722 0.9044 0.9651 - - - - -For more examples of adjoint usage on large parameter models, consult the [DiffEqFlux documentation](https://diffeqflux.sciml.ai/dev/) - -## Including Process Noise: Estimation of Stochastic Differential Equations - -This can be easily extended to Stochastic Differential Equations as well. - -Let's create the Lotka Volterra equation with some noise and try out estimating it with the same framework we have set up before. - -Our equations now become: - -$$dx = (\alpha - \beta y)xdt + \phi_1 xdW_1$$ - -$$dy = (\delta x - \gamma)ydt + \phi_2 ydW_2$$ - - -```julia -function lotka_volterra_noise(du,u,p,t) - du[1] = p[5]*u[1] - du[2] = p[6]*u[2] -end -p = [1.5, 1.0, 3.0, 1.0, 0.3, 0.3] -prob = SDEProblem(lotka_volterra,lotka_volterra_noise,u0,(0.0,10.0),p) -``` - - - - - SDEProblem with uType Array{Float64,1} and tType Float64. In-place: true - timespan: (0.0, 10.0) - u0: [1.0, 1.0] - - - -Solving it repeatedly confirms the randomness of the solution - - -```julia -sol = solve(prob,saveat=0.01) -p1 = plot(sol) -sol = solve(prob,saveat=0.01) -p2 = plot(sol) -sol = solve(prob,saveat=0.01) -p3 = plot(sol) -plot(p1,p2,p3) -``` - - - - -![svg](../10_BayesianDiffEq_files/10_BayesianDiffEq_23_0.svg) - - - -With the `MonteCarloSummary` it is easy to summarize the results from multiple runs through the `EnsembleProblem` interface, here we run the problem for 1000 `trajectories` and visualize the summary: - - -```julia -sol = solve(EnsembleProblem(prob),SRIW1(),saveat=0.1,trajectories=500) -summ = MonteCarloSummary(sol) -plot(summ) -``` - - - - -![svg](../10_BayesianDiffEq_files/10_BayesianDiffEq_25_0.svg) - - - -Get data from the means to fit: - - -```julia -using DiffEqBase.EnsembleAnalysis -averagedata = Array(timeseries_steps_mean(sol)) -``` - - - - - 2×101 Array{Float64,2}: - 1.0 1.04218 1.05885 1.03187 0.967422 … 0.190811 0.197071 0.203714 - 1.0 1.22803 1.5283 1.89036 2.30967 1.16424 1.11006 1.07984 - - - -Now fit the means with Turing. - -We will utilize multithreading with the [`EnsembleProblem`](https://docs.sciml.ai/stable/tutorials/sde_example/#Ensemble-Simulations-1) interface to speed up the SDE parameter estimation. - - -```julia -Threads.nthreads() -``` - - - - - 16 - - - - -```julia -Turing.setadbackend(:forwarddiff) - -@model function fitlv(data) - σ ~ InverseGamma(2, 3) - α ~ truncated(Normal(1.5,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.5),0,2) - γ ~ truncated(Normal(3.0,0.5),1,4) - δ ~ truncated(Normal(1.0,0.5),0,2) - ϕ1 ~ truncated(Normal(1.2,0.5),0.1,1) - ϕ2 ~ truncated(Normal(1.2,0.5),0.1,1) - - p = [α,β,γ,δ,ϕ1,ϕ2] - prob = SDEProblem(lotka_volterra,lotka_volterra_noise,u0,(0.0,10.0),p) - ensemble_predicted = solve(EnsembleProblem(prob),SRIW1(),saveat=0.1,trajectories=500) - predicted_means = timeseries_steps_mean(ensemble_predicted) - - for i = 1:length(predicted_means) - data[:,i] ~ MvNormal(predicted_means[i], σ) - end -end; - -model = fitlv(averagedata) -chain = sample(model, NUTS(.65),500) -``` - - ┌ Info: Found initial step size - │ ϵ = 0.2 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/GMBTf/src/inference/hmc.jl:629 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 0%|▏ | ETA: 0:03:49┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - Sampling: 100%|█████████████████████████████████████████| Time: 2:33:35 - - - - - - Object of type Chains, with data of type 250×19×1 Array{Float64,3} - - Iterations = 1:250 - Thinning interval = 1 - Chains = 1 - Samples per chain = 250 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ, ϕ1, ϕ2 - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ────── ────── - α 1.6255 0.0000 0.0000 0.0000 2.0325 2.5501 - β 1.1163 0.0000 0.0000 0.0000 2.0325 Inf - γ 3.2056 0.0000 0.0000 0.0000 2.0325 0.9960 - δ 0.9268 0.0000 0.0000 0.0000 2.0325 2.9880 - σ 0.0669 0.0000 0.0000 0.0000 2.0325 1.1011 - ϕ1 0.2329 0.0000 0.0000 0.0000 2.0325 3.2549 - ϕ2 0.2531 0.0000 0.0000 0.0000 2.0325 0.9960 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 1.6255 1.6255 1.6255 1.6255 1.6255 - β 1.1163 1.1163 1.1163 1.1163 1.1163 - γ 3.2056 3.2056 3.2056 3.2056 3.2056 - δ 0.9268 0.9268 0.9268 0.9268 0.9268 - σ 0.0669 0.0669 0.0669 0.0669 0.0669 - ϕ1 0.2329 0.2329 0.2329 0.2329 0.2329 - ϕ2 0.2531 0.2531 0.2531 0.2531 0.2531 - - - - - -```julia - -``` diff --git a/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_23_0.svg b/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_23_0.svg deleted file mode 100644 index 88bcd0eeb..000000000 --- a/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_23_0.svg +++ /dev/null @@ -1,922 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_25_0.svg b/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_25_0.svg deleted file mode 100644 index 41a74b51a..000000000 --- a/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_25_0.svg +++ /dev/null @@ -1,174 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_3_0.svg b/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_3_0.svg deleted file mode 100644 index aede8b899..000000000 --- a/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_3_0.svg +++ /dev/null @@ -1,314 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_9_0.svg b/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_9_0.svg deleted file mode 100644 index 77086d0ae..000000000 --- a/_tutorials/10_BayesianDiffEq_files/10_BayesianDiffEq_9_0.svg +++ /dev/null @@ -1,5398 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq.md b/_tutorials/10_diffeq.md deleted file mode 100644 index 0213b7ffb..000000000 --- a/_tutorials/10_diffeq.md +++ /dev/null @@ -1,747 +0,0 @@ ---- -title: Bayesian Estimation of Differential Equations -permalink: /:collection/:name/ ---- - -# Bayesian Estimation of Differential Equations - -Most of the scientific community deals with the basic problem of trying to mathematically model the reality around them and this often involves dynamical systems. The general trend to model these complex dynamical systems is through the use of differential equations. Differential equation models often have non-measurable parameters. The popular “forward-problem” of simulation consists of solving the differential equations for a given set of parameters, the “inverse problem” to simulation, known as parameter estimation, is the process of utilizing data to determine these model parameters. Bayesian inference provides a robust approach to parameter estimation with quantified uncertainty. - - -```julia -using Turing, Distributions, DifferentialEquations - -# Import MCMCChain, Plots, and StatsPlots for visualizations and diagnostics. -using MCMCChains, Plots, StatsPlots - -# Set a seed for reproducibility. -using Random -Random.seed!(14); - -# Disable Turing's progress meter for this tutorial. -Turing.turnprogress(false) -``` - - ┌ Info: Precompiling Turing [fce5fe82-541a-59a6-adf8-730c64b5f9a0] - └ @ Base loading.jl:1260 - ┌ Info: Precompiling DifferentialEquations [0c46a032-eb83-5123-abaf-570d42b7fbaa] - └ @ Base loading.jl:1260 - ┌ Info: Precompiling Plots [91a5bcdd-55d7-5caf-9e0b-520d859cae80] - └ @ Base loading.jl:1260 - ┌ Info: Precompiling StatsPlots [f3b207a7-027a-5e70-b257-86293d7955fd] - └ @ Base loading.jl:1260 - ┌ Info: [Turing]: progress logging is disabled globally - └ @ Turing /home/cameron/.julia/packages/Turing/GMBTf/src/Turing.jl:22 - - - - - -false - - - -Set a logger to catch AdvancedHMC warnings. - - -```julia -using Logging -Logging.disable_logging(Logging.Warn) -``` - - - - - LogLevel(1001) - - - -## The Lotka-Volterra Model - -The Lotka–Volterra equations, also known as the predator–prey equations, are a pair of first-order nonlinear differential equations, frequently used to describe the dynamics of biological systems in which two species interact, one as a predator and the other as prey. The populations change through time according to the pair of equations: - -$$\frac{dx}{dt} = (\alpha - \beta y)x$$ - -$$\frac{dy}{dt} = (\delta x - \gamma)y$$ - - - -```julia -function lotka_volterra(du,u,p,t) - x, y = u - α, β, γ, δ = p - du[1] = (α - β*y)x # dx = - du[2] = (δ*x - γ)y # dy = -end -p = [1.5, 1.0, 3.0, 1.0] -u0 = [1.0,1.0] -prob1 = ODEProblem(lotka_volterra,u0,(0.0,10.0),p) -sol = solve(prob1,Tsit5()) -plot(sol) -``` - - - - -![svg](../10_diffeq_files/10_diffeq_5_0.svg) - - - -We'll generate the data to use for the parameter estimation from simulation. -With the `saveat` [argument](https://docs.sciml.ai/latest/basics/common_solver_opts/) we specify that the solution is stored only at `0.1` time units. To make the data look more realistic, we add random noise using the function `randn`. - - -```julia -sol1 = solve(prob1,Tsit5(),saveat=0.1) -odedata = Array(sol1) + 0.8 * randn(size(Array(sol1))) -plot(sol1, alpha = 0.3, legend = false); scatter!(sol1.t, odedata') -``` - - - - -![svg](../10_diffeq_files/10_diffeq_7_0.svg) - - - -## Direct Handling of Bayesian Estimation with Turing - -Previously, functions in Turing and DifferentialEquations were not inter-composable, so Bayesian inference of differential equations needed to be handled by another package called [DiffEqBayes.jl](https://github.com/SciML/DiffEqBayes.jl) (note that DiffEqBayes works also with CmdStan.jl, Turing.jl, DynamicHMC.jl and ApproxBayes.jl - see the [DiffEqBayes docs](https://docs.sciml.ai/latest/analysis/parameter_estimation/#Bayesian-Methods-1) for more info). - -From now on however, Turing and DifferentialEquations are completely composable and we can write of the differential equation inside a Turing `@model` and it will just work. Therefore, we can rewrite the Lotka Volterra parameter estimation problem with a Turing `@model` interface as below: - - -```julia -Turing.setadbackend(:forwarddiff) - -@model function fitlv(data, prob1) - σ ~ InverseGamma(2, 3) # ~ is the tilde character - α ~ truncated(Normal(1.5,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.5),0,2) - γ ~ truncated(Normal(3.0,0.5),1,4) - δ ~ truncated(Normal(1.0,0.5),0,2) - - p = [α,β,γ,δ] - prob = remake(prob1, p=p) - predicted = solve(prob,Tsit5(),saveat=0.1) - - for i = 1:length(predicted) - data[:,i] ~ MvNormal(predicted[i], σ) - end -end - -model = fitlv(odedata, prob1) - -# This next command runs 3 independent chains without using multithreading. -chain = mapreduce(c -> sample(model, NUTS(.65),1000), chainscat, 1:3) -``` - - - - - Object of type Chains, with data of type 500×17×3 Array{Float64,3} - - Iterations = 1:500 - Thinning interval = 1 - Chains = 1, 2, 3 - Samples per chain = 500 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ────── ────── - α 1.6400 0.3788 0.0098 0.0985 6.0241 4.6765 - β 1.2568 0.4553 0.0118 0.1194 6.0241 5.6826 - γ 2.5128 1.0548 0.0272 0.2784 6.0241 6.9037 - δ 1.0579 0.4827 0.0125 0.1269 6.0241 6.1099 - σ 1.6983 0.6397 0.0165 0.1691 6.0241 7.3797 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 1.0757 1.3041 1.5536 2.0408 2.2903 - β 0.6976 0.8965 1.0570 1.8161 1.9813 - γ 1.0084 1.1320 2.8546 3.4381 3.8983 - δ 0.4722 0.5803 0.9334 1.6086 1.8763 - σ 0.7492 0.8334 2.0593 2.1754 2.3950 - - - - -The estimated parameters are close to the desired parameter values. We can also check that the chains have converged in the plot. - - -```julia -plot(chain) -``` - - - - -![svg](../10_diffeq_files/10_diffeq_12_0.svg) - - - -### Data retrodiction -In Bayesian analysis it is often useful to retrodict the data, i.e. generate simulated data using samples from the posterior distribution, and compare to the original data (see for instance section 3.3.2 - model checking of McElreath's book "Statistical Rethinking"). Here, we solve again the ODE using the output in `chain`, for 300 randomly picked posterior samples. We plot this ensemble of solutions to check if the solution resembles the data. - - -```julia -pl = scatter(sol1.t, odedata'); -``` - - -```julia -chain_array = Array(chain) -for k in 1:300 - resol = solve(remake(prob1,p=chain_array[rand(1:1500), 1:4]),Tsit5(),saveat=0.1) - plot!(resol, alpha=0.1, color = "#BBBBBB", legend = false) -end -# display(pl) -plot!(sol1, w=1, legend = false) -``` - - - - -![svg](../10_diffeq_files/10_diffeq_15_0.svg) - - - -In the plot above, the 300 retrodicted time courses from the posterior are plotted in gray, and the original data are the blue and red dots, and the solution that was used to generate the data are the green and purple lines. We can see that, even though we added quite a bit of noise to the data (see dot plot above), the posterior distribution reproduces quite accurately the "true" ODE solution. - -## Lokta Volterra with missing predator data - -Thanks to the known structure of the problem, encoded by the Lokta-Volterra ODEs, one can also fit a model with incomplete data - even without any data for one of the two variables. For instance, let's suppose you have observations for the prey only, but none for the predator. We test this case by fitting the model only to the $$y$$ variable of the system, without providing any data for $$x$$: - - -```julia -@model function fitlv2(data, prob1) # data should be a Vector - σ ~ InverseGamma(2, 3) # ~ is the tilde character - α ~ truncated(Normal(1.5,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.5),0,2) - γ ~ truncated(Normal(3.0,0.5),1,4) - δ ~ truncated(Normal(1.0,0.5),0,2) - - p = [α,β,γ,δ] - prob = remake(prob1, p=p) - predicted = solve(prob,Tsit5(),saveat=0.1) - - for i = 1:length(predicted) - data[i] ~ Normal(predicted[i][2], σ) # predicted[i][2] is the data for y - a scalar, so we use Normal instead of MvNormal - end -end - -model2 = fitlv2(odedata[2,:], prob1) -``` - - - - - DynamicPPL.Model{var"###evaluator#333",(:data, :prob1),Tuple{Array{Float64,1},ODEProblem{Array{Float64,1},Tuple{Float64,Float64},true,Array{Float64,1},ODEFunction{true,typeof(lotka_volterra),LinearAlgebra.UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}},DiffEqBase.StandardODEProblem}},(),DynamicPPL.ModelGen{var"###generator#334",(:data, :prob1),(),Tuple{}}}(##evaluator#333, (data = [1.0373159410554433, 0.455111997844011, 0.9187133767127944, -0.259115048591982, 0.3514128305537414, 0.5162643092020036, 0.9787322372445835, -0.0006805260449948558, -0.08833057357290974, 0.4636414910986264 … 3.119725220966818, 3.8955494581199934, 4.932912225131781, 2.8100177568591196, 2.925421407352717, 2.2748396927494876, 1.0152713962244975, 2.556317594971266, 2.3096409202477224, -0.30553906640714645], prob1 = ODEProblem with uType Array{Float64,1} and tType Float64. In-place: true - timespan: (0.0, 10.0) - u0: [1.0, 1.0]), DynamicPPL.ModelGen{var"###generator#334",(:data, :prob1),(),Tuple{}}(##generator#334, NamedTuple())) - - - -Here we use the multithreading functionality [available](https://turing.ml/dev/docs/using-turing/guide#multithreaded-sampling) in Turing.jl to sample 3 independent chains - - -```julia -Threads.nthreads() -``` - - - - -16 - - - - -```julia -# This next command runs 3 independent chains with multithreading. -chain2 = sample(model2, NUTS(.45), MCMCThreads(), 5000, 3, progress=false) -``` - - - - - Object of type Chains, with data of type 4000×17×3 Array{Float64,3} - - Iterations = 1:4000 - Thinning interval = 1 - Chains = 1, 2, 3 - Samples per chain = 4000 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ─────── ────── - α 1.4217 0.1660 0.0015 0.0133 51.6958 1.1826 - β 0.9862 0.1288 0.0012 0.0102 53.1975 1.1694 - γ 3.1736 0.2961 0.0027 0.0233 51.3341 1.1737 - δ 1.1284 0.2565 0.0023 0.0207 50.3184 1.1978 - σ 0.7962 0.0605 0.0006 0.0047 56.2673 1.0678 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 1.1731 1.3055 1.3872 1.5190 1.8176 - β 0.7926 0.8938 0.9629 1.0606 1.2834 - γ 2.5500 2.9797 3.2000 3.3872 3.7011 - δ 0.6123 0.9426 1.1478 1.3096 1.6027 - σ 0.6933 0.7520 0.7904 0.8358 0.9291 - - - - - -```julia -pl = scatter(sol1.t, odedata'); -chain_array2 = Array(chain2) -for k in 1:300 - resol = solve(remake(prob1,p=chain_array2[rand(1:12000), 1:4]),Tsit5(),saveat=0.1) - # Note that due to a bug in AxisArray, the variables from the chain will be returned always in - # the order it is stored in the array, not by the specified order in the call - :α, :β, :γ, :δ - plot!(resol, alpha=0.1, color = "#BBBBBB", legend = false) -end -#display(pl) -plot!(sol1, w=1, legend = false) -``` - - - - -![svg](../10_diffeq_files/10_diffeq_23_0.svg) - - - -Note that here, the data values of $$x$$ (blue dots) were not given to the model! Yet, the model could predict the values of $$x$$ relatively accurately, albeit with a wider distribution of solutions, reflecting the greater uncertainty in the prediction of the $$x$$ values. - -## Inference of Delay Differential Equations - -Here we show an example of inference with another type of differential equation: a Delay Differential Equation (DDE). A DDE is an DE system where derivatives are function of values at an earlier point in time. This is useful to model a delayed effect, like incubation time of a virus for instance. - -For this, we will define a [`DDEProblem`](https://diffeq.sciml.ai/stable/tutorials/dde_example/), from the package DifferentialEquations.jl. - -Here is a delayed version of the lokta voltera system: - -$$\frac{dx}{dt} = \alpha x(t-\tau) - \beta y(t) x(t)$$ - -$$\frac{dy}{dt} = - \gamma y(t) + \delta x(t) y(t) $$ - -Where $$x(t-\tau)$$ is the variable $$x$$ at an earlier time point. We specify the delayed variable with a function `h(p, t)`, as described in the [DDE example](https://diffeq.sciml.ai/stable/tutorials/dde_example/). - - -```julia -function delay_lotka_volterra(du, u, h, p, t) - x, y = u - α, β, γ, δ = p - du[1] = α * h(p, t-1; idxs=1) - β * x * y - du[2] = -γ * y + δ * x * y - return -end - -p = (1.5,1.0,3.0,1.0) -u0 = [1.0; 1.0] -tspan = (0.0,10.0) -h(p, t; idxs::Int) = 1.0 -prob1 = DDEProblem(delay_lotka_volterra,u0,h,tspan,p) -``` - - - - - DDEProblem with uType Array{Float64,1} and tType Float64. In-place: true - timespan: (0.0, 10.0) - u0: [1.0, 1.0] - - - - -```julia -sol = solve(prob1,saveat=0.1) -ddedata = Array(sol) -ddedata = ddedata + 0.5 * randn(size(ddedata)) -``` - - - - - 2×101 Array{Float64,2}: - 1.45377 1.01444 1.49355 1.23384 … 2.79479 2.61251 2.43377 3.17567 - 0.88201 0.214703 1.05351 0.470845 1.61454 1.31338 1.59865 0.643372 - - - -Plot the data: - - -```julia -scatter(sol.t, ddedata'); plot!(sol) -``` - - - - -![svg](../10_diffeq_files/10_diffeq_30_0.svg) - - - -Now we define and run the Turing model. - - -```julia -Turing.setadbackend(:forwarddiff) -@model function fitlv(data, prob1) - - σ ~ InverseGamma(2, 3) - α ~ Truncated(Normal(1.5,0.5),0.5,2.5) - β ~ Truncated(Normal(1.2,0.5),0,2) - γ ~ Truncated(Normal(3.0,0.5),1,4) - δ ~ Truncated(Normal(1.0,0.5),0,2) - - p = [α,β,γ,δ] - - #prob = DDEProblem(delay_lotka_volterra,u0,_h,tspan,p) - prob = remake(prob1, p=p) - predicted = solve(prob,saveat=0.1) - for i = 1:length(predicted) - data[:,i] ~ MvNormal(predicted[i], σ) - end -end; -model = fitlv(ddedata, prob1) -``` - - - - - DynamicPPL.Model{var"###evaluator#417",(:data, :prob1),Tuple{Array{Float64,2},DDEProblem{Array{Float64,1},Tuple{Float64,Float64},Tuple{},Tuple{},true,NTuple{4,Float64},DDEFunction{true,typeof(delay_lotka_volterra),LinearAlgebra.UniformScaling{Bool},Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing,Nothing},typeof(h),Base.Iterators.Pairs{Union{},Union{},Tuple{},NamedTuple{(),Tuple{}}}}},(),DynamicPPL.ModelGen{var"###generator#418",(:data, :prob1),(),Tuple{}}}(##evaluator#417, (data = [1.4537685182006146 1.0144365870095116 … 2.4337746023657396 3.175674933929347; 0.8820102262205037 0.21470268242472768 … 1.59865201888447 0.6433719617612795], prob1 = DDEProblem with uType Array{Float64,1} and tType Float64. In-place: true - timespan: (0.0, 10.0) - u0: [1.0, 1.0]), DynamicPPL.ModelGen{var"###generator#418",(:data, :prob1),(),Tuple{}}(##generator#418, NamedTuple())) - - - -Then we draw samples using multithreading; this time, we draw 3 independent chains in parallel using `MCMCThreads`. - - -```julia -chain = sample(model, NUTS(.65), MCMCThreads(), 300, 3, progress=true) -plot(chain) -``` - - - - -![svg](../10_diffeq_files/10_diffeq_34_0.svg) - - - -Finally, we select a 100 sets of parameters from the first chain and plot solutions. - - -```julia -chain -``` - - - - - Object of type Chains, with data of type 150×17×3 Array{Float64,3} - - Iterations = 1:150 - Thinning interval = 1 - Chains = 1, 2, 3 - Samples per chain = 150 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ──────── ────── - α 1.3786 0.0517 0.0024 0.0023 140.8826 1.0137 - β 0.9286 0.0444 0.0021 0.0022 237.4755 1.0020 - γ 3.2104 0.1429 0.0067 0.0083 115.0547 1.0301 - δ 1.0925 0.0512 0.0024 0.0027 122.4686 1.0216 - σ 0.4963 0.0248 0.0012 0.0012 258.5969 0.9972 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 1.2772 1.3417 1.3794 1.4082 1.4890 - β 0.8447 0.8985 0.9291 0.9543 1.0255 - γ 2.9393 3.1148 3.2113 3.2972 3.5028 - δ 1.0030 1.0568 1.0912 1.1266 1.1973 - σ 0.4518 0.4801 0.4949 0.5115 0.5480 - - - - - -```julia -pl = scatter(sol.t, ddedata') -chain_array = Array(chain) -for k in 1:100 - - resol = solve(remake(prob1,p=chain_array[rand(1:450),1:4]),Tsit5(),saveat=0.1) - # Note that due to a bug in AxisArray, the variables from the chain will be returned always in - # the order it is stored in the array, not by the specified order in the call - :α, :β, :γ, :δ - - plot!(resol, alpha=0.1, color = "#BBBBBB", legend = false) -end -#display(pl) -plot!(sol) -``` - - - - -![svg](../10_diffeq_files/10_diffeq_37_0.svg) - - - -Here again, the dots is the data fed to the model, the continuous colored line is the "true" solution, and the gray lines are solutions from the posterior. The fit is pretty good even though the data was quite noisy to start. - -## Scaling to Large Models: Adjoint Sensitivities - -DifferentialEquations.jl's efficiency for large stiff models has been shown in multiple [benchmarks](https://github.com/SciML/DiffEqBenchmarks.jl). To learn more about how to optimize solving performance for stiff problems you can take a look at the [docs](https://docs.sciml.ai/latest/tutorials/advanced_ode_example/). - -[Sensitivity analysis](https://docs.sciml.ai/latest/analysis/sensitivity/), or automatic differentiation (AD) of the solver, is provided by the DiffEq suite. The model sensitivities are the derivatives of the solution $$u(t)$$ with respect to the parameters. Specifically, the local sensitivity of the solution to a parameter is defined by how much the solution would change by changes in the parameter. Sensitivity analysis provides a cheap way to calculate the gradient of the solution which can be used in parameter estimation and other optimization tasks. - - -The AD ecosystem in Julia allows you to switch between forward mode, reverse mode, source to source and other choices of AD and have it work with any Julia code. For a user to make use of this within [SciML](https://sciml.ai), [high level interactions in `solve`](https://docs.sciml.ai/latest/analysis/sensitivity/#High-Level-Interface:-sensealg-1) automatically plug into those AD systems to allow for choosing advanced sensitivity analysis (derivative calculation) [methods](https://docs.sciml.ai/latest/analysis/sensitivity/#Sensitivity-Algorithms-1). - -More theoretical details on these methods can be found at: https://docs.sciml.ai/latest/extras/sensitivity_math/. - -While these sensitivity analysis methods may seem complicated (and they are!), using them is dead simple. Here is a version of the Lotka-Volterra model with adjoints enabled. - -All we had to do is switch the AD backend to one of the adjoint-compatible backends (ReverseDiff, Tracker, or Zygote) and boom the system takes over and we're using adjoint methods! Notice that on this model adjoints are slower. This is because adjoints have a higher overhead on small parameter models and we suggest only using these methods for models with around 100 parameters or more. For more details, see https://arxiv.org/abs/1812.01892. - - -```julia -using Zygote, DiffEqSensitivity -Turing.setadbackend(:zygote) -prob1 = ODEProblem(lotka_volterra,u0,(0.0,10.0),p) -``` - - - - - ODEProblem with uType Array{Float64,1} and tType Float64. In-place: true - timespan: (0.0, 10.0) - u0: [1.0, 1.0] - - - - -```julia -@model function fitlv(data, prob) - σ ~ InverseGamma(2, 3) - α ~ truncated(Normal(1.5,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.5),0,2) - γ ~ truncated(Normal(3.0,0.5),1,4) - δ ~ truncated(Normal(1.0,0.5),0,2) - p = [α,β,γ,δ] - prob = remake(prob, p=p) - - predicted = solve(prob,saveat=0.1) - for i = 1:length(predicted) - data[:,i] ~ MvNormal(predicted[i], σ) - end -end; -model = fitlv(odedata, prob1) -chain = sample(model, NUTS(.65),1000) -``` - - - - - Object of type Chains, with data of type 500×17×1 Array{Float64,3} - - Iterations = 1:500 - Thinning interval = 1 - Chains = 1 - Samples per chain = 500 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ────── ────── - α 2.1258 0.0002 0.0000 0.0001 3.4840 1.5025 - β 0.4210 0.0002 0.0000 0.0001 2.0647 2.9436 - γ 2.7257 0.0007 0.0000 0.0003 2.0080 2.9125 - δ 1.3201 0.0005 0.0000 0.0002 2.2723 2.3902 - σ 0.5172 0.0044 0.0002 0.0022 2.1994 2.5493 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 2.1254 2.1257 2.1258 2.1259 2.1261 - β 0.4207 0.4207 0.4209 0.4212 0.4213 - γ 2.7248 2.7250 2.7260 2.7264 2.7266 - δ 1.3193 1.3195 1.3202 1.3205 1.3208 - σ 0.5095 0.5139 0.5174 0.5210 0.5241 - - - - -Now we can exercise control of the sensitivity analysis method that is used by using the `sensealg` keyword argument. Let's choose the `InterpolatingAdjoint` from the available AD [methods](https://docs.sciml.ai/latest/analysis/sensitivity/#Sensitivity-Algorithms-1) and enable a compiled ReverseDiff vector-Jacobian product: - - -```julia -@model function fitlv(data, prob) - σ ~ InverseGamma(2, 3) - α ~ truncated(Normal(1.5,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.5),0,2) - γ ~ truncated(Normal(3.0,0.5),1,4) - δ ~ truncated(Normal(1.0,0.5),0,2) - p = [α,β,γ,δ] - prob = remake(prob, p=p) - predicted = solve(prob,saveat=0.1,sensealg=InterpolatingAdjoint(autojacvec=ReverseDiffVJP(true))) - for i = 1:length(predicted) - data[:,i] ~ MvNormal(predicted[i], σ) - end -end; -model = fitlv(odedata, prob1) -@time chain = sample(model, NUTS(.65),1000) -``` - - 476.077757 seconds (2.71 G allocations: 124.356 GiB, 5.05% gc time) - - - - - - Object of type Chains, with data of type 500×17×1 Array{Float64,3} - - Iterations = 1:500 - Thinning interval = 1 - Chains = 1 - Samples per chain = 500 - internals = acceptance_rate, hamiltonian_energy, hamiltonian_energy_error, is_accept, log_density, lp, max_hamiltonian_energy_error, n_steps, nom_step_size, numerical_error, step_size, tree_depth - parameters = α, β, γ, δ, σ - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ──────── ────── - α 1.5571 0.0535 0.0024 0.0040 87.3374 1.0194 - β 1.0574 0.0505 0.0023 0.0038 133.5800 1.0140 - γ 2.8557 0.1402 0.0063 0.0090 95.9407 1.0138 - δ 0.9331 0.0505 0.0023 0.0033 95.7488 1.0161 - σ 0.8052 0.0399 0.0018 0.0020 329.7764 0.9981 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - α 1.4561 1.5205 1.5569 1.5909 1.6727 - β 0.9631 1.0227 1.0556 1.0831 1.1629 - γ 2.5758 2.7561 2.8590 2.9456 3.1454 - δ 0.8384 0.8969 0.9327 0.9640 1.0374 - σ 0.7357 0.7767 0.8017 0.8306 0.8885 - - - - -For more examples of adjoint usage on large parameter models, consult the [DiffEqFlux documentation](https://diffeqflux.sciml.ai/dev/). - -## Inference of a Stochastic Differential Equation -A Stochastic Differential Equation ([SDE](https://diffeq.sciml.ai/stable/tutorials/sde_example/)) is a differential equation that has a stochastic (noise) term in the expression of the derivatives. Here we fit a Stochastic version of the Lokta-Volterra system. - -We use a quasi-likelihood approach in which all trajectories of a solution are compared instead of a reduction such as mean, this increases the robustness of fitting and makes the likelihood more identifiable. We use SOSRI to solve the equation. The NUTS sampler is a bit sensitive to the stochastic optimization since the gradient is then changing with every calculation, so we use NUTS with a target acceptance rate of `0.25`. - - -```julia -u0 = [1.0,1.0] -tspan = (0.0,10.0) -function multiplicative_noise!(du,u,p,t) - x,y = u - du[1] = p[5]*x - du[2] = p[6]*y -end -p = [1.5,1.0,3.0,1.0,0.1,0.1] - -function lotka_volterra!(du,u,p,t) - x,y = u - α,β,γ,δ = p - du[1] = dx = α*x - β*x*y - du[2] = dy = δ*x*y - γ*y -end - - -prob_sde = SDEProblem(lotka_volterra!,multiplicative_noise!,u0,tspan,p) - -ensembleprob = EnsembleProblem(prob_sde) -@time data = solve(ensembleprob,SOSRI(),saveat=0.1,trajectories=1000) -plot(EnsembleSummary(data)) -``` - - 15.129917 seconds (74.32 M allocations: 8.627 GiB, 8.78% gc time) - - - - - -![svg](../10_diffeq_files/10_diffeq_48_1.svg) - - - - -```julia -Turing.setadbackend(:forwarddiff) -@model function fitlv(data, prob) - σ ~ InverseGamma(2,3) - α ~ truncated(Normal(1.3,0.5),0.5,2.5) - β ~ truncated(Normal(1.2,0.25),0.5,2) - γ ~ truncated(Normal(3.2,0.25),2.2,4.0) - δ ~ truncated(Normal(1.2,0.25),0.5,2.0) - ϕ1 ~ truncated(Normal(0.12,0.3),0.05,0.25) - ϕ2 ~ truncated(Normal(0.12,0.3),0.05,0.25) - p = [α,β,γ,δ,ϕ1,ϕ2] - prob = remake(prob, p=p) - predicted = solve(prob,SOSRI(),saveat=0.1) - - if predicted.retcode != :Success - Turing.acclogp!(_varinfo, -Inf) - end - for j in 1:length(data) - for i = 1:length(predicted) - data[j][i] ~ MvNormal(predicted[i],σ) - end - end -end; -``` - -We use NUTS sampler with a low acceptance ratio and initial parameters since estimating the parameters of SDE with HMC poses a challenge. Probabilistic nature of the SDE solution makes the likelihood function noisy which poses a challenge for NUTS since the gradient is then changing with every calculation. SGHMC might be better suited to be used here. - - -```julia -model = fitlv(data, prob_sde) -chain = sample(model, NUTS(0.25), 5000, init_theta = [1.5,1.3,1.2,2.7,1.2,0.12,0.12]) -plot(chain) -``` - - - - -![svg](../10_diffeq_files/10_diffeq_51_0.svg) - - - - -```julia - -``` diff --git a/_tutorials/10_diffeq_files/10_diffeq_11_0.svg b/_tutorials/10_diffeq_files/10_diffeq_11_0.svg deleted file mode 100644 index 88f1df967..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_11_0.svg +++ /dev/null @@ -1,1920 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_12_0.svg b/_tutorials/10_diffeq_files/10_diffeq_12_0.svg deleted file mode 100644 index 335bcc2c3..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_12_0.svg +++ /dev/null @@ -1,1872 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_14_0.svg b/_tutorials/10_diffeq_files/10_diffeq_14_0.svg deleted file mode 100644 index 3e44896c7..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_14_0.svg +++ /dev/null @@ -1,8124 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_15_0.svg b/_tutorials/10_diffeq_files/10_diffeq_15_0.svg deleted file mode 100644 index 507c9ba11..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_15_0.svg +++ /dev/null @@ -1,8124 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_23_0.svg b/_tutorials/10_diffeq_files/10_diffeq_23_0.svg deleted file mode 100644 index c2d0c6cfd..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_23_0.svg +++ /dev/null @@ -1,8118 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_25_0.svg b/_tutorials/10_diffeq_files/10_diffeq_25_0.svg deleted file mode 100644 index 34b5691da..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_25_0.svg +++ /dev/null @@ -1,174 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_30_0.svg b/_tutorials/10_diffeq_files/10_diffeq_30_0.svg deleted file mode 100644 index 07e8b01dd..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_30_0.svg +++ /dev/null @@ -1,532 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_34_0.svg b/_tutorials/10_diffeq_files/10_diffeq_34_0.svg deleted file mode 100644 index 74c7fef37..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_34_0.svg +++ /dev/null @@ -1,1365 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_34_1.svg b/_tutorials/10_diffeq_files/10_diffeq_34_1.svg deleted file mode 100644 index a4e598eee..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_34_1.svg +++ /dev/null @@ -1,1341 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_37_0.svg b/_tutorials/10_diffeq_files/10_diffeq_37_0.svg deleted file mode 100644 index 36ef6ffa6..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_37_0.svg +++ /dev/null @@ -1,2918 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_3_0.svg b/_tutorials/10_diffeq_files/10_diffeq_3_0.svg deleted file mode 100644 index f70a39f32..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_3_0.svg +++ /dev/null @@ -1,314 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_47_1.svg b/_tutorials/10_diffeq_files/10_diffeq_47_1.svg deleted file mode 100644 index d2e560dcf..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_47_1.svg +++ /dev/null @@ -1,162 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_48_1.svg b/_tutorials/10_diffeq_files/10_diffeq_48_1.svg deleted file mode 100644 index f84dd84c3..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_48_1.svg +++ /dev/null @@ -1,162 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_4_0.svg b/_tutorials/10_diffeq_files/10_diffeq_4_0.svg deleted file mode 100644 index e92887c4c..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_4_0.svg +++ /dev/null @@ -1,326 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_50_1.svg b/_tutorials/10_diffeq_files/10_diffeq_50_1.svg deleted file mode 100644 index 141f79b64..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_50_1.svg +++ /dev/null @@ -1,4058 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_51_0.svg b/_tutorials/10_diffeq_files/10_diffeq_51_0.svg deleted file mode 100644 index 7313993a1..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_51_0.svg +++ /dev/null @@ -1,4082 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_5_0.svg b/_tutorials/10_diffeq_files/10_diffeq_5_0.svg deleted file mode 100644 index fb14df571..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_5_0.svg +++ /dev/null @@ -1,326 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_6_0.svg b/_tutorials/10_diffeq_files/10_diffeq_6_0.svg deleted file mode 100644 index 0fcae2e65..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_6_0.svg +++ /dev/null @@ -1,324 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_7_0.svg b/_tutorials/10_diffeq_files/10_diffeq_7_0.svg deleted file mode 100644 index 280c7f42a..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_7_0.svg +++ /dev/null @@ -1,324 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/10_diffeq_files/10_diffeq_9_0.svg b/_tutorials/10_diffeq_files/10_diffeq_9_0.svg deleted file mode 100644 index aa39227b9..000000000 --- a/_tutorials/10_diffeq_files/10_diffeq_9_0.svg +++ /dev/null @@ -1,5446 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/1_GaussianMixtureModel.md b/_tutorials/1_GaussianMixtureModel.md deleted file mode 100644 index 02a9d77bc..000000000 --- a/_tutorials/1_GaussianMixtureModel.md +++ /dev/null @@ -1,218 +0,0 @@ ---- -title: Unsupervised Learning using Bayesian Mixture Models -permalink: /:collection/:name/ ---- - -# Unsupervised Learning using Bayesian Mixture Models - -The following tutorial illustrates the use *Turing* for clustering data using a Bayesian mixture model. The aim of this task is to infer a latent grouping (hidden structure) from unlabelled data. - -More specifically, we are interested in discovering the grouping illustrated in figure below. This example consists of 2-D data points, i.e. $$\boldsymbol{x} = \{x_i\}_{i=1}^N \,, x_i \in \mathcal{R}^2$$, which are distributed according to Gaussian distributions. For simplicity, we use isotropic Gaussian distributions but this assumption can easily be relaxed by introducing additional parameters. - - -```julia -using Distributions, StatsPlots, Random - -# Set a random seed. -Random.seed!(3) - -# Construct 30 data points for each cluster. -N = 30 - -# Parameters for each cluster, we assume that each cluster is Gaussian distributed in the example. -μs = [-3.5, 0.0] - -# Construct the data points. -x = mapreduce(c -> rand(MvNormal([μs[c], μs[c]], 1.), N), hcat, 1:2) - -# Visualization. -scatter(x[1,:], x[2,:], legend = false, title = "Synthetic Dataset") -``` - - - - -![svg](../1_GaussianMixtureModel_files/1_GaussianMixtureModel_2_0.svg) - - - -## Gaussian Mixture Model in Turing - -To cluster the data points shown above, we use a model that consists of two mixture components (clusters) and assigns each datum to one of the components. The assignment thereof determines the distribution that the data point is generated from. - -In particular, in a Bayesian Gaussian mixture model with $$1 \leq k \leq K$$ components for 1-D data each data point $$x_i$$ with $$1 \leq i \leq N$$ is generated according to the following generative process. -First we draw the parameters for each cluster, i.e. in our example we draw location of the distributions from a Normal: -\$\$ -\mu_k \sim Normal() \, , \; \forall k \\ -\$\$ -and then draw mixing weight for the $$K$$ clusters from a Dirichlet distribution, i.e. -\$\$ - w \sim Dirichlet(K, \alpha) \, . \\ -\$\$ -After having constructed all the necessary model parameters, we can generate an observation by first selecting one of the clusters and then drawing the datum accordingly, i.e. -\$\$ - z_i \sim Categorical(w) \, , \; \forall i \\ - x_i \sim Normal(\mu_{z_i}, 1.) \, , \; \forall i -\$\$ - -For more details on Gaussian mixture models, we refer to Christopher M. Bishop, *Pattern Recognition and Machine Learning*, Section 9. - - -```julia -using Turing, MCMCChains - -# Turn off the progress monitor. -Turing.turnprogress(false) -``` - - ┌ Info: [Turing]: progress logging is disabled globally - └ @ Turing /home/cameron/.julia/packages/Turing/cReBm/src/Turing.jl:22 - - - - - - false - - - - -```julia -@model GaussianMixtureModel(x) = begin - - D, N = size(x) - - # Draw the parameters for cluster 1. - μ1 ~ Normal() - - # Draw the parameters for cluster 2. - μ2 ~ Normal() - - μ = [μ1, μ2] - - # Uncomment the following lines to draw the weights for the K clusters - # from a Dirichlet distribution. - - # α = 1.0 - # w ~ Dirichlet(2, α) - - # Comment out this line if you instead want to draw the weights. - w = [0.5, 0.5] - - # Draw assignments for each datum and generate it from a multivariate normal. - k = Vector{Int}(undef, N) - for i in 1:N - k[i] ~ Categorical(w) - x[:,i] ~ MvNormal([μ[k[i]], μ[k[i]]], 1.) - end - return k -end -``` - - - - - ##GaussianMixtureModel#361 (generic function with 2 methods) - - - -After having specified the model in Turing, we can construct the model function and run a MCMC simulation to obtain assignments of the data points. - - -```julia -gmm_model = GaussianMixtureModel(x); -``` - -To draw observations from the posterior distribution, we use a [particle Gibbs](https://www.stats.ox.ac.uk/~doucet/andrieu_doucet_holenstein_PMCMC.pdf) sampler to draw the discrete assignment parameters as well as a Hamiltonion Monte Carlo sampler for continous parameters. - -Note that we use a `Gibbs` sampler to combine both samplers for Bayesian inference in our model. We are also calling `mapreduce` to generate multiple chains, particularly so we test for convergence. The `chainscat` function simply adds multiple chains together. - - -```julia -gmm_sampler = Gibbs(PG(100, :k), HMC(0.05, 10, :μ1, :μ2)) -tchain = mapreduce(c -> sample(gmm_model, gmm_sampler, 100), chainscat, 1:3); -``` - -## Visualize the Density Region of the Mixture Model - -After sucessfully doing posterior inference, we can first visualize the trace and density of the parameters of interest. - -In particular, in this example we consider the sample values of the location parameter for the two clusters. - - -```julia -ids = findall(map(name -> occursin("μ", name), names(tchain))); -p=plot(tchain[:, ids, :], legend=true, labels = ["Mu 1" "Mu 2"], colordim=:parameter) -``` - - - - -![svg](../1_GaussianMixtureModel_files/1_GaussianMixtureModel_13_0.svg) - - - -You'll note here that it appears the location means are switching between chains. We will address this in future tutorials. For those who are keenly interested, see [this](https://mc-stan.org/users/documentation/case-studies/identifying_mixture_models.html) article on potential solutions. - -For the moment, we will just use the first chain to ensure the validity of our inference. - - -```julia -tchain = tchain[:, :, 1]; -``` - -As the samples for the location parameter for both clusters are unimodal, we can safely visualize the density region of our model using the average location. - - -```julia -# Helper function used for visualizing the density region. -function predict(x, y, w, μ) - # Use log-sum-exp trick for numeric stability. - return Turing.logaddexp( - log(w[1]) + logpdf(MvNormal([μ[1], μ[1]], 1.), [x, y]), - log(w[2]) + logpdf(MvNormal([μ[2], μ[2]], 1.), [x, y]) - ) -end -``` - - - - - predict (generic function with 1 method) - - - - -```julia -contour(range(-5, stop = 3), range(-6, stop = 2), - (x, y) -> predict(x, y, [0.5, 0.5], [mean(tchain[:μ1].value), mean(tchain[:μ2].value)]) -) -scatter!(x[1,:], x[2,:], legend = false, title = "Synthetic Dataset") -``` - - - - -![svg](../1_GaussianMixtureModel_files/1_GaussianMixtureModel_18_0.svg) - - - -## Infered Assignments - -Finally, we can inspect the assignments of the data points infered using Turing. As we can see, the dataset is partitioned into two distinct groups. - - -```julia -assignments = collect(skipmissing(mean(tchain[:k].value, dims=1).data)) -scatter(x[1,:], x[2,:], - legend = false, - title = "Assignments on Synthetic Dataset", - zcolor = assignments) -``` - - - - -![svg](../1_GaussianMixtureModel_files/1_GaussianMixtureModel_21_0.svg) - - diff --git a/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_13_0.svg b/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_13_0.svg deleted file mode 100644 index 304b23100..000000000 --- a/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_13_0.svg +++ /dev/null @@ -1,982 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -25 - - -50 - - -75 - - -100 - - --3 - - --2 - - --1 - - -0 - - -Chain 1 - - -Iteration - - -Sample value - - - - - - - -Mu 1 - - - -Mu 2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --3 - - --2 - - --1 - - -0 - - -0.0 - - -0.5 - - -1.0 - - -1.5 - - -2.0 - - -2.5 - - -Chain 1 - - -Sample value - - -Density - - - - - - - -Mu 1 - - - -Mu 2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -25 - - -50 - - -75 - - -100 - - --3 - - --2 - - --1 - - -0 - - -Chain 2 - - -Iteration - - -Sample value - - - - - - - -Mu 1 - - - -Mu 2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --3 - - --2 - - --1 - - -0 - - -0 - - -1 - - -2 - - -3 - - -Chain 2 - - -Sample value - - -Density - - - - - - - -Mu 1 - - - -Mu 2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -25 - - -50 - - -75 - - -100 - - --3 - - --2 - - --1 - - -0 - - -Chain 3 - - -Iteration - - -Sample value - - - - - - - -Mu 1 - - - -Mu 2 - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --3 - - --2 - - --1 - - -0 - - -0 - - -1 - - -2 - - -3 - - -Chain 3 - - -Sample value - - -Density - - - - - - - -Mu 1 - - - -Mu 2 - - diff --git a/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_18_0.svg b/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_18_0.svg deleted file mode 100644 index f7542a338..000000000 --- a/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_18_0.svg +++ /dev/null @@ -1,369 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --2 - - -0 - - -2 - - --6 - - --4 - - --2 - - -0 - - -2 - - -Synthetic Dataset - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -- - - -25.0 - - -- - - -22.5 - - -- - - -20.0 - - -- - - -17.5 - - -- - - -15.0 - - -- - - -12.5 - - -- - - -10.0 - - -- - - -7.5 - - -- - - -5.0 - - - diff --git a/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_21_0.svg b/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_21_0.svg deleted file mode 100644 index f4a0597af..000000000 --- a/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_21_0.svg +++ /dev/null @@ -1,165 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --2 - - -0 - - -2 - - --6 - - --4 - - --2 - - -0 - - -Assignments on Synthetic Dataset - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_2_0.svg b/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_2_0.svg deleted file mode 100644 index 61c910d89..000000000 --- a/_tutorials/1_GaussianMixtureModel_files/1_GaussianMixtureModel_2_0.svg +++ /dev/null @@ -1,165 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --2 - - -0 - - -2 - - --6 - - --4 - - --2 - - -0 - - -Synthetic Dataset - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/2_LogisticRegression.md b/_tutorials/2_LogisticRegression.md deleted file mode 100644 index 2dae9f505..000000000 --- a/_tutorials/2_LogisticRegression.md +++ /dev/null @@ -1,312 +0,0 @@ ---- -title: Bayesian Logistic Regression -permalink: /:collection/:name/ ---- - -# Bayesian Logistic Regression -[Bayesian logistic regression](https://en.wikipedia.org/wiki/Logistic_regression#Bayesian) is the Bayesian counterpart to a common tool in machine learning, logistic regression. The goal of logistic regression is to predict a one or a zero for a given training item. An example might be predicting whether someone is sick or ill given their symptoms and personal information. - -In our example, we'll be working to predict whether someone is likely to default with a synthetic dataset found in the `RDatasets` package. This dataset, `Defaults`, comes from R's [ISLR](https://cran.r-project.org/web/packages/ISLR/index.html) package and contains information on borrowers. - -To start, let's import all the libraries we'll need. - - -```julia -# Import Turing and Distributions. -using Turing, Distributions - -# Import RDatasets. -using RDatasets - -# Import MCMCChains, Plots, and StatsPlots for visualizations and diagnostics. -using MCMCChains, Plots, StatsPlots - -# We need a logistic function, which is provided by StatsFuns. -using StatsFuns: logistic - -# Functionality for splitting and normalizing the data -using MLDataUtils: shuffleobs, stratifiedobs, rescale! - -# Set a seed for reproducibility. -using Random -Random.seed!(0); - -# Turn off progress monitor. -Turing.turnprogress(false) -``` - - ┌ Info: [Turing]: progress logging is disabled globally - └ @ Turing /home/cameron/.julia/packages/Turing/cReBm/src/Turing.jl:22 - - - - - - false - - - -## Data Cleaning & Set Up - -Now we're going to import our dataset. The first six rows of the dataset are shown below so you can get a good feel for what kind of data we have. - - -```julia -# Import the "Default" dataset. -data = RDatasets.dataset("ISLR", "Default"); - -# Show the first six rows of the dataset. -first(data, 6) -``` - - - - -

6 rows × 4 columns

DefaultStudentBalanceIncome
Categorical…Categorical…Float64Float64
1NoNo729.52644361.6
2NoYes817.1812106.1
3NoNo1073.5531767.1
4NoNo529.25135704.5
5NoNo785.65638463.5
6NoYes919.5897491.56
- - - -Most machine learning processes require some effort to tidy up the data, and this is no different. We need to convert the `Default` and `Student` columns, which say "Yes" or "No" into 1s and 0s. Afterwards, we'll get rid of the old words-based columns. - - -```julia -# Convert "Default" and "Student" to numeric values. -data[!,:DefaultNum] = [r.Default == "Yes" ? 1.0 : 0.0 for r in eachrow(data)] -data[!,:StudentNum] = [r.Student == "Yes" ? 1.0 : 0.0 for r in eachrow(data)] - -# Delete the old columns which say "Yes" and "No". -select!(data, Not([:Default, :Student])) - -# Show the first six rows of our edited dataset. -first(data, 6) -``` - - - - -

6 rows × 4 columns

BalanceIncomeDefaultNumStudentNum
Float64Float64Float64Float64
1729.52644361.60.00.0
2817.1812106.10.01.0
31073.5531767.10.00.0
4529.25135704.50.00.0
5785.65638463.50.00.0
6919.5897491.560.01.0
- - - -After we've done that tidying, it's time to split our dataset into training and testing sets, and separate the labels from the data. We separate our data into two halves, `train` and `test`. You can use a higher percentage of splitting (or a lower one) by modifying the `at = 0.05` argument. We have highlighted the use of only a 5% sample to show the power of Bayesian inference with small sample sizes. - -We must rescale our variables so that they are centered around zero by subtracting each column by the mean and dividing it by the standard deviation. Without this step, Turing's sampler will have a hard time finding a place to start searching for parameter estimates. To do this we will leverage `MLDataUtils`, which also lets us effortlessly shuffle our observations and perform a stratified split to get a representative test set. - - -```julia -function split_data(df, target; at = 0.70) - shuffled = shuffleobs(df) - trainset, testset = stratifiedobs(row -> row[target], - shuffled, p = at) -end - -features = [:StudentNum, :Balance, :Income] -numerics = [:Balance, :Income] -target = :DefaultNum - -trainset, testset = split_data(data, target, at = 0.05) -for feature in numerics - μ, σ = rescale!(trainset[!, feature], obsdim=1) - rescale!(testset[!, feature], μ, σ, obsdim=1) -end - -# Turing requires data in matrix form, not dataframe -train = Matrix(trainset[:, features]) -test = Matrix(testset[:, features]) -train_label = trainset[:, target] -test_label = testset[:, target]; -``` - -## Model Declaration -Finally, we can define our model. - -`logistic_regression` takes four arguments: - -- `x` is our set of independent variables; -- `y` is the element we want to predict; -- `n` is the number of observations we have; and -- `σ` is the standard deviation we want to assume for our priors. - -Within the model, we create four coefficients (`intercept`, `student`, `balance`, and `income`) and assign a prior of normally distributed with means of zero and standard deviations of `σ`. We want to find values of these four coefficients to predict any given `y`. - -The `for` block creates a variable `v` which is the logistic function. We then observe the liklihood of calculating `v` given the actual label, `y[i]`. - - -```julia -# Bayesian logistic regression (LR) -@model logistic_regression(x, y, n, σ) = begin - intercept ~ Normal(0, σ) - - student ~ Normal(0, σ) - balance ~ Normal(0, σ) - income ~ Normal(0, σ) - - for i = 1:n - v = logistic(intercept + student*x[i, 1] + balance*x[i,2] + income*x[i,3]) - y[i] ~ Bernoulli(v) - end -end; -``` - -## Sampling - -Now we can run our sampler. This time we'll use [`HMC`](http://turing.ml/docs/library/#Turing.HMC) to sample from our posterior. - - -```julia -# Retrieve the number of observations. -n, _ = size(train) - -# Sample using HMC. -chain = mapreduce(c -> sample(logistic_regression(train, train_label, n, 1), HMC(0.05, 10), 1500), - chainscat, - 1:3 -) - -describe(chain) -``` - - - - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ─────── ────── ──────── ────── ───────── ────── - balance 1.6517 0.3099 0.0046 0.0080 110.2122 1.0004 - income -0.5174 0.3241 0.0048 0.0081 1440.4337 1.0010 - intercept -3.8265 0.5148 0.0077 0.0148 54.8792 1.0004 - student -1.8662 0.6088 0.0091 0.0223 840.9122 1.0037 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ─────── ─────── ─────── ─────── ─────── - balance 1.1418 1.4534 1.6331 1.8242 2.2196 - income -1.1678 -0.7300 -0.5094 -0.3006 0.1079 - intercept -4.6202 -4.0685 -3.7947 -3.5465 -3.0855 - student -3.0690 -2.2803 -1.8574 -1.4528 -0.7137 - - - - -Since we ran multiple chains, we may as well do a spot check to make sure each chain converges around similar points. - - -```julia -plot(chain) -``` - - - - -![svg](../2_LogisticRegression_files/2_LogisticRegression_13_0.svg) - - - -Looks good! - -We can also use the `corner` function from MCMCChains to show the distributions of the various parameters of our logistic regression. - - -```julia -# The labels to use. -l = [:student, :balance, :income] - -# Use the corner function. Requires StatsPlots and MCMCChains. -corner(chain, l) -``` - - - - -![svg](../2_LogisticRegression_files/2_LogisticRegression_15_0.svg) - - - -Fortunately the corner plot appears to demonstrate unimodal distributions for each of our parameters, so it should be straightforward to take the means of each parameter's sampled values to estimate our model to make predictions. - -## Making Predictions -How do we test how well the model actually predicts whether someone is likely to default? We need to build a prediction function that takes the `test` object we made earlier and runs it through the average parameter calculated during sampling. - -The `prediction` function below takes a `Matrix` and a `Chain` object. It takes the mean of each parameter's sampled values and re-runs the logistic function using those mean values for every element in the test set. - - -```julia -function prediction(x::Matrix, chain, threshold) - # Pull the means from each parameter's sampled values in the chain. - intercept = mean(chain[:intercept].value) - student = mean(chain[:student].value) - balance = mean(chain[:balance].value) - income = mean(chain[:income].value) - - # Retrieve the number of rows. - n, _ = size(x) - - # Generate a vector to store our predictions. - v = Vector{Float64}(undef, n) - - # Calculate the logistic function for each element in the test set. - for i in 1:n - num = logistic(intercept .+ student * x[i,1] + balance * x[i,2] + income * x[i,3]) - if num >= threshold - v[i] = 1 - else - v[i] = 0 - end - end - return v -end; -``` - -Let's see how we did! We run the test matrix through the prediction function, and compute the [mean squared error](https://en.wikipedia.org/wiki/Mean_squared_error) (MSE) for our prediction. The `threshold` variable sets the sensitivity of the predictions. For example, a threshold of 0.07 will predict a defualt value of 1 for any predicted value greater than 0.07 and no default if it is less than 0.07. - - -```julia -# Set the prediction threshold. -threshold = 0.07 - -# Make the predictions. -predictions = prediction(test, chain, threshold) - -# Calculate MSE for our test set. -loss = sum((predictions - test_label).^2) / length(test_label) -``` - - - - - 0.1163157894736842 - - - -Perhaps more important is to see what percentage of defaults we correctly predicted. The code below simply counts defaults and predictions and presents the results. - - -```julia -defaults = sum(test_label) -not_defaults = length(test_label) - defaults - -predicted_defaults = sum(test_label .== predictions .== 1) -predicted_not_defaults = sum(test_label .== predictions .== 0) - -println("Defaults: $$defaults - Predictions: $$predicted_defaults - Percentage defaults correct $$(predicted_defaults/defaults)") - -println("Not defaults: $$not_defaults - Predictions: $$predicted_not_defaults - Percentage non-defaults correct $$(predicted_not_defaults/not_defaults)") -``` - - Defaults: 316.0 - Predictions: 265 - Percentage defaults correct 0.8386075949367089 - Not defaults: 9184.0 - Predictions: 8130 - Percentage non-defaults correct 0.8852351916376306 - - -The above shows that with a threshold of 0.07, we correctly predict a respectable portion of the defaults, and correctly identify most non-defaults. This is fairly sensitive to a choice of threshold, and you may wish to experiment with it. - -This tutorial has demonstrated how to use Turing to perform Bayesian logistic regression. diff --git a/_tutorials/2_LogisticRegression_files/2_LogisticRegression_13_0.svg b/_tutorials/2_LogisticRegression_files/2_LogisticRegression_13_0.svg deleted file mode 100644 index 8ced50970..000000000 --- a/_tutorials/2_LogisticRegression_files/2_LogisticRegression_13_0.svg +++ /dev/null @@ -1,3095 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - -1 - - -2 - - -3 - - -4 - - -5 - - -6 - - -balance - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 - - -2 - - -3 - - -4 - - -5 - - -6 - - -7 - - -0.0 - - -0.5 - - -1.0 - - -1.5 - - -balance - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --2.5 - - --2.0 - - --1.5 - - --1.0 - - --0.5 - - -0.0 - - -0.5 - - -income - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - --1 - - -0 - - -1 - - -0.0 - - -0.2 - - -0.4 - - -0.6 - - -0.8 - - -1.0 - - -1.0 - - -income - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --16 - - --14 - - --12 - - --10 - - --8 - - --6 - - --4 - - -intercept - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --15 - - --12 - - --9 - - --6 - - --3 - - -0.00 - - -0.25 - - -0.50 - - -0.75 - - -1.00 - - -intercept - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --4 - - --3 - - --2 - - --1 - - -0 - - -student - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5 - - --4 - - --3 - - --2 - - --1 - - -0 - - -1 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -student - - -Sample value - - -Density - - - - - diff --git a/_tutorials/2_LogisticRegression_files/2_LogisticRegression_15_0.svg b/_tutorials/2_LogisticRegression_files/2_LogisticRegression_15_0.svg deleted file mode 100644 index 9b485a728..000000000 --- a/_tutorials/2_LogisticRegression_files/2_LogisticRegression_15_0.svg +++ /dev/null @@ -1,15705 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - --1 - - -0 - - -1 - - -balance - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 - - -2 - - -3 - - -4 - - -5 - - -6 - - --5 - - --4 - - --3 - - --2 - - --1 - - -0 - - -student - - -income - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - --1 - - -0 - - -1 - - -balance - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/3_BayesNN.md b/_tutorials/3_BayesNN.md deleted file mode 100644 index 41ab7bbd4..000000000 --- a/_tutorials/3_BayesNN.md +++ /dev/null @@ -1,412 +0,0 @@ ---- -title: Bayesian Neural Networks -permalink: /:collection/:name/ ---- - -# Bayesian Neural Networks - -In this tutorial, we demonstrate how one can implement a Bayesian Neural Network using a combination of Turing and [Flux](https://github.com/FluxML/Flux.jl), a suite of tools machine learning. We will use Flux to specify the neural network's layers and Turing to implement the probabalistic inference, with the goal of implementing a classification algorithm. - -We will begin with importing the relevant libraries. - - -```julia -# Import libraries. -using Turing, Flux, Plots, Random - -# Hide sampling progress. -Turing.turnprogress(false); - -# Use reverse_diff due to the number of parameters in neural networks. -Turing.setadbackend(:reversediff) -``` - - ┌ Info: [Turing]: progress logging is disabled globally - └ @ Turing /home/cameron/.julia/packages/Turing/cReBm/src/Turing.jl:22 - - - - - - :reversediff - - - -Our goal here is to use a Bayesian neural network to classify points in an artificial dataset. The code below generates data points arranged in a box-like pattern and displays a graph of the dataset we'll be working with. - - -```julia -# Number of points to generate. -N = 80 -M = round(Int, N / 4) -Random.seed!(1234) - -# Generate artificial data. -x1s = rand(M) * 4.5; x2s = rand(M) * 4.5; -xt1s = Array([[x1s[i] + 0.5; x2s[i] + 0.5] for i = 1:M]) -x1s = rand(M) * 4.5; x2s = rand(M) * 4.5; -append!(xt1s, Array([[x1s[i] - 5; x2s[i] - 5] for i = 1:M])) - -x1s = rand(M) * 4.5; x2s = rand(M) * 4.5; -xt0s = Array([[x1s[i] + 0.5; x2s[i] - 5] for i = 1:M]) -x1s = rand(M) * 4.5; x2s = rand(M) * 4.5; -append!(xt0s, Array([[x1s[i] - 5; x2s[i] + 0.5] for i = 1:M])) - -# Store all the data for later. -xs = [xt1s; xt0s] -ts = [ones(2*M); zeros(2*M)] - -# Plot data points. -function plot_data() - x1 = map(e -> e[1], xt1s) - y1 = map(e -> e[2], xt1s) - x2 = map(e -> e[1], xt0s) - y2 = map(e -> e[2], xt0s) - - Plots.scatter(x1,y1, color="red", clim = (0,1)) - Plots.scatter!(x2, y2, color="blue", clim = (0,1)) -end - -plot_data() -``` - - - - -![svg](../3_BayesNN_files/3_BayesNN_4_0.svg) - - - -## Building a Neural Network - -The next step is to define a [feedforward neural network](https://en.wikipedia.org/wiki/Feedforward_neural_network) where we express our parameters as distribtuions, and not single points as with traditional neural networks. The two functions below, `unpack` and `nn_forward` are helper functions we need when we specify our model in Turing. - -`unpack` takes a vector of parameters and partitions them between weights and biases. `nn_forward` constructs a neural network with the variables generated in `unpack` and returns a prediction based on the weights provided. - -The `unpack` and `nn_forward` functions are explicity designed to create a neural network with two hidden layers and one output layer, as shown below. - -nn-diagram - -The end of this tutorial provides some code that can be used to generate more general network shapes. - - -```julia -# Turn a vector into a set of weights and biases. -function unpack(nn_params::AbstractVector) - W₁ = reshape(nn_params[1:6], 3, 2); - b₁ = reshape(nn_params[7:9], 3) - - W₂ = reshape(nn_params[10:15], 2, 3); - b₂ = reshape(nn_params[16:17], 2) - - Wₒ = reshape(nn_params[18:19], 1, 2); - bₒ = reshape(nn_params[20:20], 1) - return W₁, b₁, W₂, b₂, Wₒ, bₒ -end - -# Construct a neural network using Flux and return a predicted value. -function nn_forward(xs, nn_params::AbstractVector) - W₁, b₁, W₂, b₂, Wₒ, bₒ = unpack(nn_params) - nn = Chain(Dense(W₁, b₁, tanh), - Dense(W₂, b₂, tanh), - Dense(Wₒ, bₒ, σ)) - return nn(xs) -end; -``` - -The probabalistic model specification below creates a `params` variable, which has 20 normally distributed variables. Each entry in the `params` vector represents weights and biases of our neural net. - - -```julia -# Create a regularization term and a Gaussain prior variance term. -alpha = 0.09 -sig = sqrt(1.0 / alpha) - -# Specify the probabalistic model. -@model bayes_nn(xs, ts) = begin - # Create the weight and bias vector. - nn_params ~ MvNormal(zeros(20), sig .* ones(20)) - - # Calculate predictions for the inputs given the weights - # and biases in theta. - preds = nn_forward(xs, nn_params) - - # Observe each prediction. - for i = 1:length(ts) - ts[i] ~ Bernoulli(preds[i]) - end -end; -``` - -Inference can now be performed by calling `sample`. We use the `HMC` sampler here. - - -```julia -# Perform inference. -N = 5000 -ch = sample(bayes_nn(hcat(xs...), ts), HMC(0.05, 4), N); -``` - -Now we extract the weights and biases from the sampled chain. We'll use these primarily in determining how good a classifier our model is. - - -```julia -# Extract all weight and bias parameters. -theta = ch[:nn_params].value.data; -``` - -## Prediction Visualization - -We can use [MAP estimation](https://en.wikipedia.org/wiki/Maximum_a_posteriori_estimation) to classify our population by using the set of weights that provided the highest log posterior. - - -```julia -# Plot the data we have. -plot_data() - -# Find the index that provided the highest log posterior in the chain. -_, i = findmax(ch[:lp].value.data) - -# Extract the max row value from i. -i = i.I[1] - -# Plot the posterior distribution with a contour plot. -x_range = collect(range(-6,stop=6,length=25)) -y_range = collect(range(-6,stop=6,length=25)) -Z = [nn_forward([x, y], theta[i, :])[1] for x=x_range, y=y_range] -contour!(x_range, y_range, Z) -``` - - - - -![svg](../3_BayesNN_files/3_BayesNN_16_0.svg) - - - -The contour plot above shows that the MAP method is not too bad at classifying our data. - -Now we can visualize our predictions. - -\$\$ -p(\tilde{x} | X, \alpha) = \int_{\theta} p(\tilde{x} | \theta) p(\theta | X, \alpha) \approx \sum_{\theta \sim p(\theta | X, \alpha)}f_{\theta}(\tilde{x}) -\$\$ - -The `nn_predict` function takes the average predicted value from a network parameterized by weights drawn from the MCMC chain. - - -```julia -# Return the average predicted value across -# multiple weights. -function nn_predict(x, theta, num) - mean([nn_forward(x, theta[i,:])[1] for i in 1:10:num]) -end; -``` - -Next, we use the `nn_predict` function to predict the value at a sample of points where the `x` and `y` coordinates range between -6 and 6. As we can see below, we still have a satisfactory fit to our data. - - -```julia -# Plot the average prediction. -plot_data() - -n_end = 1500 -x_range = collect(range(-6,stop=6,length=25)) -y_range = collect(range(-6,stop=6,length=25)) -Z = [nn_predict([x, y], theta, n_end)[1] for x=x_range, y=y_range] -contour!(x_range, y_range, Z) -``` - - - - -![svg](../3_BayesNN_files/3_BayesNN_21_0.svg) - - - -If you are interested in how the predictive power of our Bayesian neural network evolved between samples, the following graph displays an animation of the contour plot generated from the network weights in samples 1 to 1,000. - - -```julia -# Number of iterations to plot. -n_end = 500 - -anim = @gif for i=1:n_end - plot_data() - Z = [nn_forward([x, y], theta[i,:])[1] for x=x_range, y=y_range] - contour!(x_range, y_range, Z, title="Iteration $$i", clim = (0,1)) -end every 5 - - -``` - - ┌ Info: Saved animation to - │ fn = /home/cameron/code/TuringTutorials/tmp.gif - └ @ Plots /home/cameron/.julia/packages/Plots/cc8wh/src/animation.jl:98 - - - - - - - - - -## Variational Inference (ADVI) - -We can also use Turing's variational inference tools to estimate the parameters of this model. See [variational inference](https://turing.ml/dev/docs/for-developers/variational_inference) for more information. - - -```julia -using Bijectors -using Turing: Variational - -m = bayes_nn(hcat(xs...), ts); - -q = Variational.meanfield(m) - -μ = randn(length(q)) -ω = -1 .* ones(length(q)) - -q = Variational.update(q, μ, exp.(ω)); - -advi = ADVI(10, 1000) -q_hat = vi(m, advi, q); -``` - - ┌ Info: [ADVI] Should only be seen once: optimizer created for θ - │ objectid(θ) = 3812708583762184342 - └ @ Turing.Variational /home/cameron/.julia/packages/Turing/cReBm/src/variational/VariationalInference.jl:204 - - - -```julia -samples = transpose(rand(q_hat, 5000)) -ch_vi = Chains(reshape(samples, size(samples)..., 1), ["nn_params[$$i]" for i = 1:20]); - -# Extract all weight and bias parameters. -theta = ch_vi[:nn_params].value.data; -``` - - -```julia -# Plot the average prediction. -plot_data() - -n_end = 1500 -x_range = collect(range(-6,stop=6,length=25)) -y_range = collect(range(-6,stop=6,length=25)) -Z = [nn_predict([x, y], theta, n_end)[1] for x=x_range, y=y_range] -contour!(x_range, y_range, Z) -``` - - - - -![svg](../3_BayesNN_files/3_BayesNN_28_0.svg) - - - -## Generic Bayesian Neural Networks - -The below code is intended for use in more general applications, where you need to be able to change the basic network shape fluidly. The code above is highly rigid, and adapting it for other architectures would be time consuming. Currently the code below only supports networks of `Dense` layers. - -Here, we solve the same problem as above, but with three additional 2x2 `tanh` hidden layers. You can modify the `network_shape` variable to specify differing architectures. A tuple `(3,2, :tanh)` means you want to construct a `Dense` layer with 3 outputs, 2 inputs, and a `tanh` activation function. You can provide any activation function found in Flux by entering it as a `Symbol` (e.g., the `tanh` function is entered in the third part of the tuple as `:tanh`). - - -```julia -# Specify the network architecture. -network_shape = [ - (3,2, :tanh), - (2,3, :tanh), - (1,2, :σ)] - -# Regularization, parameter variance, and total number of -# parameters. -alpha = 0.09 -sig = sqrt(1.0 / alpha) -num_params = sum([i * o + i for (i, o, _) in network_shape]) - -# This modification of the unpack function generates a series of vectors -# given a network shape. -function unpack(θ::AbstractVector, network_shape::AbstractVector) - index = 1 - weights = [] - biases = [] - for layer in network_shape - rows, cols, _ = layer - size = rows * cols - last_index_w = size + index - 1 - last_index_b = last_index_w + rows - push!(weights, reshape(θ[index:last_index_w], rows, cols)) - push!(biases, reshape(θ[last_index_w+1:last_index_b], rows)) - index = last_index_b + 1 - end - return weights, biases -end - -# Generate an abstract neural network given a shape, -# and return a prediction. -function nn_forward(x, θ::AbstractVector, network_shape::AbstractVector) - weights, biases = unpack(θ, network_shape) - layers = [] - for i in eachindex(network_shape) - push!(layers, Dense(weights[i], - biases[i], - eval(network_shape[i][3]))) - end - nn = Chain(layers...) - return nn(x) -end - -# General Turing specification for a BNN model. -@model bayes_nn_general(xs, ts, network_shape, num_params) = begin - θ ~ MvNormal(zeros(num_params), sig .* ones(num_params)) - preds = nn_forward(xs, θ, network_shape) - for i = 1:length(ts) - ts[i] ~ Bernoulli(preds[i]) - end -end - -# Set the backend. -Turing.setadbackend(:reverse_diff) - -# Perform inference. -num_samples = 500 -ch2 = sample(bayes_nn_general(hcat(xs...), ts, network_shape, num_params), NUTS(0.65), num_samples); -``` - - ┌ Warning: `Turing.setadbackend(:reverse_diff)` is deprecated. Please use `Turing.setadbackend(:tracker)` to use `Tracker` or `Turing.setadbackend(:reversediff)` to use `ReverseDiff`. To use `ReverseDiff`, please make sure it is loaded separately with `using ReverseDiff`. - │ caller = setadbackend(::Symbol) at ad.jl:5 - └ @ Turing.Core /home/cameron/.julia/packages/Turing/cReBm/src/core/ad.jl:5 - ┌ Info: Found initial step size - │ ϵ = 0.2 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/cReBm/src/inference/hmc.jl:556 - - - -```julia -# This function makes predictions based on network shape. -function nn_predict(x, theta, num, network_shape) - mean([nn_forward(x, theta[i,:], network_shape)[1] for i in 1:10:num]) -end; - -# Extract the θ parameters from the sampled chain. -params2 = ch2[:θ].value.data - -plot_data() - -x_range = collect(range(-6,stop=6,length=25)) -y_range = collect(range(-6,stop=6,length=25)) -Z = [nn_predict([x, y], params2, length(ch2), network_shape)[1] for x=x_range, y=y_range] -contour!(x_range, y_range, Z) -``` - - - - -![svg](../3_BayesNN_files/3_BayesNN_31_0.svg) - - - -This has been an introduction to the applications of Turing and Flux in defining Bayesian neural networks. diff --git a/_tutorials/3_BayesNN_files/3_BayesNN_16_0.svg b/_tutorials/3_BayesNN_files/3_BayesNN_16_0.svg deleted file mode 100644 index 179549d9e..000000000 --- a/_tutorials/3_BayesNN_files/3_BayesNN_16_0.svg +++ /dev/null @@ -1,453 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - -y1 - - - -y2 - - diff --git a/_tutorials/3_BayesNN_files/3_BayesNN_21_0.svg b/_tutorials/3_BayesNN_files/3_BayesNN_21_0.svg deleted file mode 100644 index c5d7f8b43..000000000 --- a/_tutorials/3_BayesNN_files/3_BayesNN_21_0.svg +++ /dev/null @@ -1,458 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - -y1 - - - -y2 - - diff --git a/_tutorials/3_BayesNN_files/3_BayesNN_28_0.svg b/_tutorials/3_BayesNN_files/3_BayesNN_28_0.svg deleted file mode 100644 index c7af91efb..000000000 --- a/_tutorials/3_BayesNN_files/3_BayesNN_28_0.svg +++ /dev/null @@ -1,334 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - -y1 - - - -y2 - - diff --git a/_tutorials/3_BayesNN_files/3_BayesNN_31_0.svg b/_tutorials/3_BayesNN_files/3_BayesNN_31_0.svg deleted file mode 100644 index 374e9cae0..000000000 --- a/_tutorials/3_BayesNN_files/3_BayesNN_31_0.svg +++ /dev/null @@ -1,454 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - -y1 - - - -y2 - - diff --git a/_tutorials/3_BayesNN_files/3_BayesNN_32_0.svg b/_tutorials/3_BayesNN_files/3_BayesNN_32_0.svg deleted file mode 100644 index 0d58b42aa..000000000 --- a/_tutorials/3_BayesNN_files/3_BayesNN_32_0.svg +++ /dev/null @@ -1,538 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - - -y1 - - - - -y2 - - diff --git a/_tutorials/3_BayesNN_files/3_BayesNN_35_0.svg b/_tutorials/3_BayesNN_files/3_BayesNN_35_0.svg deleted file mode 100644 index f2118f6ad..000000000 --- a/_tutorials/3_BayesNN_files/3_BayesNN_35_0.svg +++ /dev/null @@ -1,553 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - - -y1 - - - - -y2 - - diff --git a/_tutorials/3_BayesNN_files/3_BayesNN_4_0.svg b/_tutorials/3_BayesNN_files/3_BayesNN_4_0.svg deleted file mode 100644 index 33e69f84f..000000000 --- a/_tutorials/3_BayesNN_files/3_BayesNN_4_0.svg +++ /dev/null @@ -1,214 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --2 - - -0 - - -2 - - -4 - - --4 - - --2 - - -0 - - -2 - - -4 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -y1 - - - -y2 - - diff --git a/_tutorials/4_BayesHmm.md b/_tutorials/4_BayesHmm.md deleted file mode 100644 index 7ffa7cf41..000000000 --- a/_tutorials/4_BayesHmm.md +++ /dev/null @@ -1,200 +0,0 @@ ---- -title: Bayesian Hidden Markov Models -permalink: /:collection/:name/ ---- - -# Bayesian Hidden Markov Models -This tutorial illustrates training Bayesian [Hidden Markov Models](https://en.wikipedia.org/wiki/Hidden_Markov_model) (HMM) using Turing. The main goals are learning the transition matrix, emission parameter, and hidden states. For a more rigorous academic overview on Hidden Markov Models, see [An introduction to Hidden Markov Models and Bayesian Networks](http://mlg.eng.cam.ac.uk/zoubin/papers/ijprai.pdf) (Ghahramani, 2001). - -Let's load the libraries we'll need. We also set a random seed (for reproducibility) and the automatic differentiation backend to forward mode (more [here](http://turing.ml/docs/autodiff/) on why this is useful). - - -```julia -# Load libraries. -using Turing, Plots, Random - -# Turn off progress monitor. -Turing.turnprogress(false); - -# Set a random seed and use the forward_diff AD mode. -Random.seed!(1234); -``` - - ┌ Info: [Turing]: progress logging is disabled globally - └ @ Turing /home/cameron/.julia/packages/Turing/cReBm/src/Turing.jl:22 - - -## Simple State Detection - -In this example, we'll use something where the states and emission parameters are straightforward. - - -```julia -# Define the emission parameter. -y = [ 1.0, 1.0, 1.0, 1.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 2.0, 2.0, 2.0, 1.0, 1.0 ]; -N = length(y); K = 3; - -# Plot the data we just made. -plot(y, xlim = (0,15), ylim = (-1,5), size = (500, 250)) -``` - - - - -![svg](../4_BayesHmm_files/4_BayesHmm_3_0.svg) - - - -We can see that we have three states, one for each height of the plot (1, 2, 3). This height is also our emission parameter, so state one produces a value of one, state two produces a value of two, and so on. - -Ultimately, we would like to understand three major parameters: - -1. The transition matrix. This is a matrix that assigns a probability of switching from one state to any other state, including the state that we are already in. -2. The emission matrix, which describes a typical value emitted by some state. In the plot above, the emission parameter for state one is simply one. -3. The state sequence is our understanding of what state we were actually in when we observed some data. This is very important in more sophisticated HMM models, where the emission value does not equal our state. - -With this in mind, let's set up our model. We are going to use some of our knowledge as modelers to provide additional information about our system. This takes the form of the prior on our emission parameter. - -\$\$ -m_i \sim Normal(i, 0.5), \space m = \{1,2,3\} -\$\$ - -Simply put, this says that we expect state one to emit values in a Normally distributed manner, where the mean of each state's emissions is that state's value. The variance of 0.5 helps the model converge more quickly — consider the case where we have a variance of 1 or 2. In this case, the likelihood of observing a 2 when we are in state 1 is actually quite high, as it is within a standard deviation of the true emission value. Applying the prior that we are likely to be tightly centered around the mean prevents our model from being too confused about the state that is generating our observations. - -The priors on our transition matrix are noninformative, using `T[i] ~ Dirichlet(ones(K)/K)`. The Dirichlet prior used in this way assumes that the state is likely to change to any other state with equal probability. As we'll see, this transition matrix prior will be overwritten as we observe data. - - -```julia -# Turing model definition. -@model BayesHmm(y, K) = begin - # Get observation length. - N = length(y) - - # State sequence. - s = tzeros(Int, N) - - # Emission matrix. - m = Vector(undef, K) - - # Transition matrix. - T = Vector{Vector}(undef, K) - - # Assign distributions to each element - # of the transition matrix and the - # emission matrix. - for i = 1:K - T[i] ~ Dirichlet(ones(K)/K) - m[i] ~ Normal(i, 0.5) - end - - # Observe each point of the input. - s[1] ~ Categorical(K) - y[1] ~ Normal(m[s[1]], 0.1) - - for i = 2:N - s[i] ~ Categorical(vec(T[s[i-1]])) - y[i] ~ Normal(m[s[i]], 0.1) - end -end; -``` - -We will use a combination of two samplers ([HMC](http://turing.ml/docs/library/#Turing.HMC) and [Particle Gibbs](http://turing.ml/docs/library/#Turing.PG)) by passing them to the [Gibbs](http://turing.ml/docs/library/#Turing.Gibbs) sampler. The Gibbs sampler allows for compositional inference, where we can utilize different samplers on different parameters. - -In this case, we use HMC for `m` and `T`, representing the emission and transition matrices respectively. We use the Particle Gibbs sampler for `s`, the state sequence. You may wonder why it is that we are not assigning `s` to the HMC sampler, and why it is that we need compositional Gibbs sampling at all. - -The parameter `s` is not a continuous variable. It is a vector of **integers**, and thus Hamiltonian methods like HMC and [NUTS](http://turing.ml/docs/library/#-turingnuts--type) won't work correctly. Gibbs allows us to apply the right tools to the best effect. If you are a particularly advanced user interested in higher performance, you may benefit from setting up your Gibbs sampler to use [different automatic differentiation](http://turing.ml/docs/autodiff/#compositional-sampling-with-differing-ad-modes) backends for each parameter space. - -Time to run our sampler. - - -```julia -g = Gibbs(HMC(0.001, 7, :m, :T), PG(20, :s)) -c = sample(BayesHmm(y, 3), g, 100); -``` - -Let's see how well our chain performed. Ordinarily, using the `describe` function from [MCMCChain](https://github.com/TuringLang/MCMCChain.jl) would be a good first step, but we have generated a lot of parameters here (`s[1]`, `s[2]`, `m[1]`, and so on). It's a bit easier to show how our model performed graphically. - -The code below generates an animation showing the graph of the data above, and the data our model generates in each sample. - - -```julia -# Import StatsPlots for animating purposes. -using StatsPlots - -# Extract our m and s parameters from the chain. -m_set = c[:m].value.data -s_set = c[:s].value.data - -# Iterate through the MCMC samples. -Ns = 1:length(c) - -# Make an animation. -animation = @animate for i in Ns - m = m_set[i, :]; - s = Int.(s_set[i,:]); - emissions = collect(skipmissing(m[s])) - - p = plot(y, c = :red, - size = (500, 250), - xlabel = "Time", - ylabel = "State", - legend = :topright, label = "True data", - xlim = (0,15), - ylim = (-1,5)); - plot!(emissions, color = :blue, label = "Sample $$N") -end every 10; -``` - -![animation](https://user-images.githubusercontent.com/422990/50612436-de588980-0e8e-11e9-8635-4e3e97c0d7f9.gif) - -Looks like our model did a pretty good job, but we should also check to make sure our chain converges. A quick check is to examine whether the diagonal (representing the probability of remaining in the current state) of the transition matrix appears to be stationary. The code below extracts the diagonal and shows a traceplot of each persistence probability. - - -```julia -# Index the chain with the persistence probabilities. -subchain = c[:,["T[$$i][$$i]" for i in 1:K],:] - -# Plot the chain. -plot(subchain, - colordim = :parameter, - seriestype=:traceplot, - title = "Persistence Probability", - legend=:right - ) -``` - - - - -![svg](../4_BayesHmm_files/4_BayesHmm_11_0.svg) - - - -A cursory examination of the traceplot above indicates that at least `T[3,3]` and possibly `T[2,2]` have converged to something resembling stationary. `T[1,1]`, on the other hand, has a slight "wobble", and seems less consistent than the others. We can use the diagnostic functions provided by [MCMCChain](https://github.com/TuringLang/MCMCChain.jl) to engage in some formal tests, like the Heidelberg and Welch diagnostic: - - -```julia -heideldiag(c[:T]) -``` - - - - - 1-element Array{ChainDataFrame{NamedTuple{(:parameters, Symbol("Burn-in"), :Stationarity, Symbol("p-value"), :Mean, :Halfwidth, :Test),Tuple{Array{String,1},Array{Float64,1},Array{Float64,1},Array{Float64,1},Array{Float64,1},Array{Float64,1},Array{Float64,1}}}},1}: - Heidelberger and Welch Diagnostic - Chain 1 - parameters Burn-in Stationarity p-value Mean Halfwidth Test - ────────── ─────── ──────────── ─────── ────── ───────── ────── - T[1][1] 50.0000 0.0000 0.0001 0.5329 0.0063 1.0000 - T[1][2] 50.0000 0.0000 0.0189 0.1291 0.0043 1.0000 - T[1][3] 50.0000 0.0000 0.0230 0.3381 0.0032 1.0000 - T[2][1] 30.0000 1.0000 0.2757 0.0037 0.0000 1.0000 - T[2][2] 0.0000 1.0000 0.1689 0.0707 0.0022 1.0000 - T[2][3] 0.0000 1.0000 0.1365 0.9255 0.0022 1.0000 - T[3][1] 50.0000 0.0000 0.0454 0.4177 0.0147 1.0000 - T[3][2] 40.0000 1.0000 0.0909 0.2549 0.0080 1.0000 - T[3][3] 50.0000 0.0000 0.0098 0.3274 0.0067 1.0000 - - - - -The p-values on the test suggest that we cannot reject the hypothesis that the observed sequence comes from a stationary distribution, so we can be somewhat more confident that our transition matrix has converged to something reasonable. diff --git a/_tutorials/4_BayesHmm_files/4_BayesHmm_11_0.svg b/_tutorials/4_BayesHmm_files/4_BayesHmm_11_0.svg deleted file mode 100644 index a57ff9bc2..000000000 --- a/_tutorials/4_BayesHmm_files/4_BayesHmm_11_0.svg +++ /dev/null @@ -1,192 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -25 - - -50 - - -75 - - -100 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -Persistence Probability - - -Iteration - - -Sample value - - - - - - - - -["T[1][1]", "T[2][2]", "T[3][3]"] - - - -["T[1][1]", "T[2][2]", "T[3][3]"] - - - -["T[1][1]", "T[2][2]", "T[3][3]"] - - diff --git a/_tutorials/4_BayesHmm_files/4_BayesHmm_3_0.svg b/_tutorials/4_BayesHmm_files/4_BayesHmm_3_0.svg deleted file mode 100644 index f34e10e66..000000000 --- a/_tutorials/4_BayesHmm_files/4_BayesHmm_3_0.svg +++ /dev/null @@ -1,163 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.0 - - -2.5 - - -5.0 - - -7.5 - - -10.0 - - -12.5 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -5 - - - - - - -y1 - - diff --git a/_tutorials/5_LinearRegression.md b/_tutorials/5_LinearRegression.md deleted file mode 100644 index 0f78fe421..000000000 --- a/_tutorials/5_LinearRegression.md +++ /dev/null @@ -1,343 +0,0 @@ ---- -title: Linear Regression -permalink: /:collection/:name/ ---- - -# Linear Regression - -Turing is powerful when applied to complex hierarchical models, but it can also be put to task at common statistical procedures, like [linear regression](https://en.wikipedia.org/wiki/Linear_regression). This tutorial covers how to implement a linear regression model in Turing. - -## Set Up - -We begin by importing all the necessary libraries. - - -```julia -# Import Turing and Distributions. -using Turing, Distributions - -# Import RDatasets. -using RDatasets - -# Import MCMCChains, Plots, and StatPlots for visualizations and diagnostics. -using MCMCChains, Plots, StatsPlots - -# Functionality for splitting and normalizing the data. -using MLDataUtils: shuffleobs, splitobs, rescale! - -# Functionality for evaluating the model predictions. -using Distances - -# Set a seed for reproducibility. -using Random -Random.seed!(0) - -# Hide the progress prompt while sampling. -Turing.turnprogress(false); -``` - - ┌ Info: Precompiling Turing [fce5fe82-541a-59a6-adf8-730c64b5f9a0] - └ @ Base loading.jl:1260 - ┌ Info: Precompiling RDatasets [ce6b1742-4840-55fa-b093-852dadbb1d8b] - └ @ Base loading.jl:1260 - ┌ Info: Precompiling Plots [91a5bcdd-55d7-5caf-9e0b-520d859cae80] - └ @ Base loading.jl:1260 - ┌ Info: Precompiling StatsPlots [f3b207a7-027a-5e70-b257-86293d7955fd] - └ @ Base loading.jl:1260 - ┌ Info: Precompiling MLDataUtils [cc2ba9b6-d476-5e6d-8eaf-a92d5412d41d] - └ @ Base loading.jl:1260 - ┌ Info: [Turing]: progress logging is disabled globally - └ @ Turing /home/cameron/.julia/packages/Turing/GMBTf/src/Turing.jl:22 - - -We will use the `mtcars` dataset from the [RDatasets](https://github.com/johnmyleswhite/RDatasets.jl) package. `mtcars` contains a variety of statistics on different car models, including their miles per gallon, number of cylinders, and horsepower, among others. - -We want to know if we can construct a Bayesian linear regression model to predict the miles per gallon of a car, given the other statistics it has. Lets take a look at the data we have. - - -```julia -# Import the "Default" dataset. -data = RDatasets.dataset("datasets", "mtcars"); - -# Show the first six rows of the dataset. -first(data, 6) -``` - - - - -

6 rows × 12 columns (omitted printing of 3 columns)

ModelMPGCylDispHPDRatWTQSecVS
StringFloat64Int64Float64Int64Float64Float64Float64Int64
1Mazda RX421.06160.01103.92.6216.460
2Mazda RX4 Wag21.06160.01103.92.87517.020
3Datsun 71022.84108.0933.852.3218.611
4Hornet 4 Drive21.46258.01103.083.21519.441
5Hornet Sportabout18.78360.01753.153.4417.020
6Valiant18.16225.01052.763.4620.221
- - - - -```julia -size(data) -``` - - - - - (32, 12) - - - -The next step is to get our data ready for testing. We'll split the `mtcars` dataset into two subsets, one for training our model and one for evaluating our model. Then, we separate the targets we want to learn (`MPG`, in this case) and standardize the datasets by subtracting each column's means and dividing by the standard deviation of that column. The resulting data is not very familiar looking, but this standardization process helps the sampler converge far easier. - - -```julia -# Remove the model column. -select!(data, Not(:Model)) - -# Split our dataset 70%/30% into training/test sets. -trainset, testset = splitobs(shuffleobs(data), 0.7) - -# Turing requires data in matrix form. -target = :MPG -train = Matrix(select(trainset, Not(target))) -test = Matrix(select(testset, Not(target))) -train_target = trainset[:, target] -test_target = testset[:, target] - -# Standardize the features. -μ, σ = rescale!(train; obsdim = 1) -rescale!(test, μ, σ; obsdim = 1) - -# Standardize the targets. -μtarget, σtarget = rescale!(train_target; obsdim = 1) -rescale!(test_target, μtarget, σtarget; obsdim = 1); -``` - -## Model Specification - -In a traditional frequentist model using [OLS](https://en.wikipedia.org/wiki/Ordinary_least_squares), our model might look like: - -\$\$ -MPG_i = \alpha + \boldsymbol{\beta}^\mathsf{T}\boldsymbol{X_i} -\$\$ - -where $$\boldsymbol{\beta}$$ is a vector of coefficients and $$\boldsymbol{X}$$ is a vector of inputs for observation $$i$$. The Bayesian model we are more concerned with is the following: - -\$\$ -MPG_i \sim \mathcal{N}(\alpha + \boldsymbol{\beta}^\mathsf{T}\boldsymbol{X_i}, \sigma^2) -\$\$ - -where $$\alpha$$ is an intercept term common to all observations, $$\boldsymbol{\beta}$$ is a coefficient vector, $$\boldsymbol{X_i}$$ is the observed data for car $$i$$, and $$\sigma^2$$ is a common variance term. - -For $$\sigma^2$$, we assign a prior of `truncated(Normal(0, 100), 0, Inf)`. This is consistent with [Andrew Gelman's recommendations](http://www.stat.columbia.edu/~gelman/research/published/taumain.pdf) on noninformative priors for variance. The intercept term ($$\alpha$$) is assumed to be normally distributed with a mean of zero and a variance of three. This represents our assumptions that miles per gallon can be explained mostly by our assorted variables, but a high variance term indicates our uncertainty about that. Each coefficient is assumed to be normally distributed with a mean of zero and a variance of 10. We do not know that our coefficients are different from zero, and we don't know which ones are likely to be the most important, so the variance term is quite high. Lastly, each observation $$y_i$$ is distributed according to the calculated `mu` term given by $$\alpha + \boldsymbol{\beta}^\mathsf{T}\boldsymbol{X_i}$$. - - -```julia -# Bayesian linear regression. -@model function linear_regression(x, y) - # Set variance prior. - σ₂ ~ truncated(Normal(0, 100), 0, Inf) - - # Set intercept prior. - intercept ~ Normal(0, sqrt(3)) - - # Set the priors on our coefficients. - nfeatures = size(x, 2) - coefficients ~ MvNormal(nfeatures, sqrt(10)) - - # Calculate all the mu terms. - mu = intercept .+ x * coefficients - y ~ MvNormal(mu, sqrt(σ₂)) -end -``` - - - - - DynamicPPL.ModelGen{var"###generator#273",(:x, :y),(),Tuple{}}(##generator#273, NamedTuple()) - - - -With our model specified, we can call the sampler. We will use the No U-Turn Sampler ([NUTS](http://turing.ml/docs/library/#-turingnuts--type)) here. - - -```julia -model = linear_regression(train, train_target) -chain = sample(model, NUTS(0.65), 3_000); -``` - - ┌ Info: Found initial step size - │ ϵ = 1.6 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/GMBTf/src/inference/hmc.jl:629 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/P9wqk/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/P9wqk/src/hamiltonian.jl:47 - - -As a visual check to confirm that our coefficients have converged, we show the densities and trace plots for our parameters using the `plot` functionality. - - -```julia -plot(chain) -``` - - - - -![svg](../5_LinearRegression_files/5_LinearRegression_12_0.svg) - - - -It looks like each of our parameters has converged. We can check our numerical esimates using `describe(chain)`, as below. - - -```julia -describe(chain) -``` - - - - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ──────────────── ─────── ────── ──────── ────── ──────── ────── - coefficients[1] -0.0413 0.5648 0.0126 0.0389 265.1907 1.0010 - coefficients[2] 0.2770 0.6994 0.0156 0.0401 375.2777 1.0067 - coefficients[3] -0.4116 0.3850 0.0086 0.0160 695.3990 1.0032 - coefficients[4] 0.1805 0.2948 0.0066 0.0126 479.9290 1.0010 - coefficients[5] -0.2669 0.7168 0.0160 0.0316 373.0291 1.0009 - coefficients[6] 0.0256 0.3461 0.0077 0.0119 571.0954 1.0028 - coefficients[7] 0.0277 0.3899 0.0087 0.0174 637.1596 1.0007 - coefficients[8] 0.1535 0.3050 0.0068 0.0117 579.1998 1.0032 - coefficients[9] 0.1223 0.2839 0.0063 0.0105 587.6752 0.9995 - coefficients[10] -0.2839 0.3975 0.0089 0.0195 360.9612 1.0019 - intercept 0.0058 0.1179 0.0026 0.0044 580.0222 0.9995 - σ₂ 0.3017 0.1955 0.0044 0.0132 227.2322 1.0005 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ──────────────── ─────── ─────── ─────── ─────── ────── - coefficients[1] -1.0991 -0.4265 -0.0199 0.3244 1.1093 - coefficients[2] -1.1369 -0.1523 0.2854 0.7154 1.6488 - coefficients[3] -1.1957 -0.6272 -0.3986 -0.1800 0.3587 - coefficients[4] -0.3896 -0.0155 0.1663 0.3593 0.7818 - coefficients[5] -1.6858 -0.6835 -0.2683 0.1378 1.1995 - coefficients[6] -0.6865 -0.1672 0.0325 0.2214 0.7251 - coefficients[7] -0.7644 -0.1976 0.0090 0.2835 0.8185 - coefficients[8] -0.4980 -0.0194 0.1451 0.3428 0.7685 - coefficients[9] -0.4643 -0.0294 0.1237 0.2807 0.7218 - coefficients[10] -1.0898 -0.5091 -0.2846 -0.0413 0.5163 - intercept -0.2240 -0.0671 0.0083 0.0746 0.2364 - σ₂ 0.1043 0.1860 0.2525 0.3530 0.8490 - - - - -## Comparing to OLS - -A satisfactory test of our model is to evaluate how well it predicts. Importantly, we want to compare our model to existing tools like OLS. The code below uses the [GLM.jl]() package to generate a traditional OLS multiple regression model on the same data as our probabalistic model. - - -```julia -# Import the GLM package. -using GLM - -# Perform multiple regression OLS. -train_with_intercept = hcat(ones(size(train, 1)), train) -ols = lm(train_with_intercept, train_target) - -# Compute predictions on the training data set -# and unstandardize them. -p = GLM.predict(ols) -train_prediction_ols = μtarget .+ σtarget .* p - -# Compute predictions on the test data set -# and unstandardize them. -test_with_intercept = hcat(ones(size(test, 1)), test) -p = GLM.predict(ols, test_with_intercept) -test_prediction_ols = μtarget .+ σtarget .* p; -``` - - ┌ Info: Precompiling GLM [38e38edf-8417-5370-95a0-9cbb8c7f171a] - └ @ Base loading.jl:1260 - - -The function below accepts a chain and an input matrix and calculates predictions. We use the samples of the model parameters in the chain starting with sample 200, which is where the warm-up period for the NUTS sampler ended. - - -```julia -# Make a prediction given an input vector. -function prediction(chain, x) - p = get_params(chain[200:end, :, :]) - targets = p.intercept' .+ x * reduce(hcat, p.coefficients)' - return vec(mean(targets; dims = 2)) -end -``` - - - - - prediction (generic function with 1 method) - - - -When we make predictions, we unstandardize them so they are more understandable. - - -```julia -# Calculate the predictions for the training and testing sets -# and unstandardize them. -p = prediction(chain, train) -train_prediction_bayes = μtarget .+ σtarget .* p -p = prediction(chain, test) -test_prediction_bayes = μtarget .+ σtarget .* p - -# Show the predictions on the test data set. -DataFrame( - MPG = testset[!, target], - Bayes = test_prediction_bayes, - OLS = test_prediction_ols -) -``` - - - - -

10 rows × 3 columns

MPGBayesOLS
Float64Float64Float64
119.218.376618.1265
215.06.41766.37891
316.413.912513.883
414.311.839311.7337
521.425.362225.1916
618.120.768720.672
719.716.0315.8408
815.218.290318.3391
926.028.519128.4865
1017.314.49814.534
- - - -Now let's evaluate the loss for each method, and each prediction set. We will use the mean squared error to evaluate loss, given by -\$\$ -\text{MSE} = \frac{1}{n} \sum_{i=1}^n {(y_i - \hat{y_i})^2} -\$\$ -where $$y_i$$ is the actual value (true MPG) and $$\hat{y_i}$$ is the predicted value using either OLS or Bayesian linear regression. A lower SSE indicates a closer fit to the data. - - -```julia -println( - "Training set:", - "\n\tBayes loss: ", - msd(train_prediction_bayes, trainset[!, target]), - "\n\tOLS loss: ", - msd(train_prediction_ols, trainset[!, target]) -) - -println( - "Test set:", - "\n\tBayes loss: ", - msd(test_prediction_bayes, testset[!, target]), - "\n\tOLS loss: ", - msd(test_prediction_ols, testset[!, target]) -) -``` - - Training set: - Bayes loss: 4.664508273535872 - OLS loss: 4.648142085690519 - Test set: - Bayes loss: 14.66153554719035 - OLS loss: 14.796847779051628 - - -As we can see above, OLS and our Bayesian model fit our training and test data set about the same. diff --git a/_tutorials/5_LinearRegression_files/5_LinearRegression_12_0.svg b/_tutorials/5_LinearRegression_files/5_LinearRegression_12_0.svg deleted file mode 100644 index 1d26e9819..000000000 --- a/_tutorials/5_LinearRegression_files/5_LinearRegression_12_0.svg +++ /dev/null @@ -1,4546 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/6_InfiniteMixtureModel.md b/_tutorials/6_InfiniteMixtureModel.md deleted file mode 100644 index 84b147b85..000000000 --- a/_tutorials/6_InfiniteMixtureModel.md +++ /dev/null @@ -1,315 +0,0 @@ ---- -title: Probabilistic Modelling using the Infinite Mixture Model -permalink: /:collection/:name/ ---- - -# Probabilistic Modelling using the Infinite Mixture Model - -In many applications it is desirable to allow the model to adjust its complexity to the amount the data. Consider for example the task of assigning objects into clusters or groups. This task often involves the specification of the number of groups. However, often times it is not known beforehand how many groups exist. Moreover, in some applictions, e.g. modelling topics in text documents or grouping species, the number of examples per group is heavy tailed. This makes it impossible to predefine the number of groups and requiring the model to form new groups when data points from previously unseen groups are observed. - -A natural approach for such applications is the use of non-parametric models. This tutorial will introduce how to use the Dirichlet process in a mixture of infinitely many Gaussians using Turing. For further information on Bayesian nonparametrics and the Dirichlet process we refer to the [introduction by Zoubin Ghahramani](http://mlg.eng.cam.ac.uk/pub/pdf/Gha12.pdf) and the book "Fundamentals of Nonparametric Bayesian Inference" by Subhashis Ghosal and Aad van der Vaart. - - -```julia -using Turing -``` - -## Mixture Model - -Before introducing infinite mixture models in Turing, we will briefly review the construction of finite mixture models. Subsequently, we will define how to use the [Chinese restaurant process](https://en.wikipedia.org/wiki/Chinese_restaurant_process) construction of a Dirichlet process for non-parametric clustering. - -#### Two-Component Model - -First, consider the simple case of a mixture model with two Gaussian components with fixed covariance. -The generative process of such a model can be written as: - -$$ -\begin{align} -\pi_1 &\sim Beta(a, b) \\ -\pi_2 &= 1-\pi_1 \\ -\mu_1 &\sim Normal(\mu_0, \Sigma_0) \\ -\mu_2 &\sim Normal(\mu_0, \Sigma_0) \\ -z_i &\sim Categorical(\pi_1, \pi_2) \\ -x_i &\sim Normal(\mu_{z_i}, \Sigma) -\end{align} -$$ - -where $$\pi_1, \pi_2$$ are the mixing weights of the mixture model, i.e. $$\pi_1 + \pi_2 = 1$$, and $$z_i$$ is a latent assignment of the observation $$x_i$$ to a component (Gaussian). - -We can implement this model in Turing for 1D data as follows: - - -```julia -@model two_model(x) = begin - - # Hyper-parameters - μ0 = 0.0 - σ0 = 1.0 - - # Draw weights. - π1 ~ Beta(1,1) - π2 = 1-π1 - - # Draw locations of the components. - μ1 ~ Normal(μ0, σ0) - μ2 ~ Normal(μ0, σ0) - - # Draw latent assignment. - z ~ Categorical([π1, π2]) - - # Draw observation from selected component. - if z == 1 - x ~ Normal(μ1, 1.0) - else - x ~ Normal(μ2, 1.0) - end -end -``` - - - - - DynamicPPL.ModelGen{var"###generator#282",(:x,),(),Tuple{}}(##generator#282, NamedTuple()) - - - -#### Finite Mixture Model - -If we have more than two components, this model can elegantly be extend using a Dirichlet distribution as prior for the mixing weights $$\pi_1, \dots, \pi_K$$. Note that the Dirichlet distribution is the multivariate generalization of the beta distribution. The resulting model can be written as: - -$$ -\begin{align} -(\pi_1, \dots, \pi_K) &\sim Dirichlet(K, \alpha) \\ -\mu_k &\sim Normal(\mu_0, \Sigma_0), \;\; \forall k \\ -z &\sim Categorical(\pi_1, \dots, \pi_K) \\ -x &\sim Normal(\mu_z, \Sigma) -\end{align} -$$ - -which resembles the model in the [Gaussian mixture model tutorial](https://turing.ml/dev/tutorials/1-gaussianmixturemodel/) with a slightly different notation. - -## Infinite Mixture Model - -The question now arises, is there a generalization of a Dirichlet distribution for which the dimensionality $$K$$ is infinite, i.e. $$K = \infty$$? - -But first, to implement an infinite Gaussian mixture model in Turing, we first need to load the `Turing.RandomMeasures` module. `RandomMeasures` contains a variety of tools useful in nonparametrics. - - -```julia -using Turing.RandomMeasures -``` - -We now will utilize the fact that one can integrate out the mixing weights in a Gaussian mixture model allowing us to arrive at the Chinese restaurant process construction. See Carl E. Rasmussen: [The Infinite Gaussian Mixture Model](https://www.seas.harvard.edu/courses/cs281/papers/rasmussen-1999a.pdf), NIPS (2000) for details. - -In fact, if the mixing weights are integrated out, the conditional prior for the latent variable $$z$$ is given by: - -\$\$ -p(z_i = k \mid z_{\not i}, \alpha) = \frac{n_k + \alpha/K}{N - 1 + \alpha} -\$\$ - -where $$z_{\not i}$$ are the latent assignments of all observations except observation $$i$$. Note that we use $$n_k$$ to denote the number of observations at component $$k$$ excluding observation $$i$$. The parameter $$\alpha$$ is the concentration parameter of the Dirichlet distribution used as prior over the mixing weights. - -#### Chinese Restaurant Process - -To obtain the Chinese restaurant process construction, we can now derive the conditional prior if $$K \rightarrow \infty$$. - -For $$n_k > 0$$ we obtain: - -\$\$ -p(z_i = k \mid z_{\not i}, \alpha) = \frac{n_k}{N - 1 + \alpha} -\$\$ - -and for all infinitely many clusters that are empty (combined) we get: - -\$\$ -p(z_i = k \mid z_{\not i}, \alpha) = \frac{\alpha}{N - 1 + \alpha} -\$\$ - - -Those equations show that the conditional prior for component assignments is proportional to the number of such observations, meaning that the Chinese restaurant process has a rich get richer property. - -To get a better understanding of this property, we can plot the cluster choosen by for each new observation drawn from the conditional prior. - - -```julia -# Concentration parameter. -α = 10.0 - -# Random measure, e.g. Dirichlet process. -rpm = DirichletProcess(α) - -# Cluster assignments for each observation. -z = Vector{Int}() - -# Maximum number of observations we observe. -Nmax = 500 - -for i in 1:Nmax - # Number of observations per cluster. - K = isempty(z) ? 0 : maximum(z) - nk = Vector{Int}(map(k -> sum(z .== k), 1:K)) - - # Draw new assignment. - push!(z, rand(ChineseRestaurantProcess(rpm, nk))) -end -``` - - -```julia -using Plots - -# Plot the cluster assignments over time -@gif for i in 1:Nmax - scatter(collect(1:i), z[1:i], markersize = 2, xlabel = "observation (i)", ylabel = "cluster (k)", legend = false) -end; -``` - - ┌ Info: Saved animation to - │ fn = /home/cameron/code/TuringTutorials/tmp.gif - └ @ Plots /home/cameron/.julia/packages/Plots/Xnzc7/src/animation.jl:104 - - -![tmp](https://user-images.githubusercontent.com/422990/55284032-90cfa980-5323-11e9-8a99-f9315db170cb.gif) - -Further, we can see that the number of clusters is logarithmic in the number of observations and data points. This is a side-effect of the "rich-get-richer" phenomenon, i.e. we expect large clusters and thus the number of clusters has to be smaller than the number of observations. - -\$\$ -E[K \mid N] \approx \alpha \cdot log \big(1 + \frac{N}{\alpha}\big) -\$\$ - -We can see from the equation that the concentration parameter $$\alpha$$ allows us to control the number of clusters formed *a priori*. - -In Turing we can implement an infinite Gaussian mixture model using the Chinese restaurant process construction of a Dirichlet process as follows: - - -```julia -@model infiniteGMM(x) = begin - - # Hyper-parameters, i.e. concentration parameter and parameters of H. - α = 1.0 - μ0 = 0.0 - σ0 = 1.0 - - # Define random measure, e.g. Dirichlet process. - rpm = DirichletProcess(α) - - # Define the base distribution, i.e. expected value of the Dirichlet process. - H = Normal(μ0, σ0) - - # Latent assignment. - z = tzeros(Int, length(x)) - - # Locations of the infinitely many clusters. - μ = tzeros(Float64, 0) - - for i in 1:length(x) - - # Number of clusters. - K = maximum(z) - nk = Vector{Int}(map(k -> sum(z .== k), 1:K)) - - # Draw the latent assignment. - z[i] ~ ChineseRestaurantProcess(rpm, nk) - - # Create a new cluster? - if z[i] > K - push!(μ, 0.0) - - # Draw location of new cluster. - μ[z[i]] ~ H - end - - # Draw observation. - x[i] ~ Normal(μ[z[i]], 1.0) - end -end -``` - - - - - DynamicPPL.ModelGen{var"###generator#800",(:x,),(),Tuple{}}(##generator#800, NamedTuple()) - - - -We can now use Turing to infer the assignments of some data points. First, we will create some random data that comes from three clusters, with means of 0, -5, and 10. - - -```julia -using Plots, Random - -# Generate some test data. -Random.seed!(1) -data = vcat(randn(10), randn(10) .- 5, randn(10) .+ 10) -data .-= mean(data) -data /= std(data); -``` - -Next, we'll sample from our posterior using SMC. - - -```julia -# MCMC sampling -Random.seed!(2) -iterations = 1000 -model_fun = infiniteGMM(data); -chain = sample(model_fun, SMC(), iterations); -``` - - Sampling: 100%|█████████████████████████████████████████| Time: 0:00:00 - - -Finally, we can plot the number of clusters in each sample. - - -```julia -# Extract the number of clusters for each sample of the Markov chain. -k = map(t -> length(unique(chain[:z].value[t,:,:])), 1:iterations); - -# Visualize the number of clusters. -plot(k, xlabel = "Iteration", ylabel = "Number of clusters", label = "Chain 1") -``` - - - - -![svg](../6_InfiniteMixtureModel_files/6_InfiniteMixtureModel_31_0.svg) - - - -If we visualize the histogram of the number of clusters sampled from our posterior, we observe that the model seems to prefer 3 clusters, which is the true number of clusters. Note that the number of clusters in a Dirichlet process mixture model is not limited a priori and will grow to infinity with probability one. However, if conditioned on data the posterior will concentrate on a finite number of clusters enforcing the resulting model to have a finite amount of clusters. It is, however, not given that the posterior of a Dirichlet process Gaussian mixture model converges to the true number of clusters, given that data comes from a finite mixture model. See Jeffrey Miller and Matthew Harrison: [A simple example of Dirichlet process mixture inconsitency for the number of components](https://arxiv.org/pdf/1301.2708.pdf) for details. - - -```julia -histogram(k, xlabel = "Number of clusters", legend = false) -``` - - - - -![svg](../6_InfiniteMixtureModel_files/6_InfiniteMixtureModel_33_0.svg) - - - -One issue with the Chinese restaurant process construction is that the number of latent parameters we need to sample scales with the number of observations. It may be desirable to use alternative constructions in certain cases. Alternative methods of constructing a Dirichlet process can be employed via the following representations: - -Size-Biased Sampling Process - -\$\$ -j_k \sim Beta(1, \alpha) * surplus -\$\$ - -Stick-Breaking Process -\$\$ -v_k \sim Beta(1, \alpha) -\$\$ - -Chinese Restaurant Process -\$\$ -p(z_n = k | z_{1:n-1}) \propto \begin{cases} - \frac{m_k}{n-1+\alpha}, \text{ if } m_k > 0\\\ - \frac{\alpha}{n-1+\alpha} - \end{cases} -\$\$ - -For more details see [this article](https://www.stats.ox.ac.uk/~teh/research/npbayes/Teh2010a.pdf). diff --git a/_tutorials/6_InfiniteMixtureModel_files/6_InfiniteMixtureModel_31_0.svg b/_tutorials/6_InfiniteMixtureModel_files/6_InfiniteMixtureModel_31_0.svg deleted file mode 100644 index 889394978..000000000 --- a/_tutorials/6_InfiniteMixtureModel_files/6_InfiniteMixtureModel_31_0.svg +++ /dev/null @@ -1,196 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/6_InfiniteMixtureModel_files/6_InfiniteMixtureModel_33_0.svg b/_tutorials/6_InfiniteMixtureModel_files/6_InfiniteMixtureModel_33_0.svg deleted file mode 100644 index 83df27173..000000000 --- a/_tutorials/6_InfiniteMixtureModel_files/6_InfiniteMixtureModel_33_0.svg +++ /dev/null @@ -1,180 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/7_PoissonRegression.md b/_tutorials/7_PoissonRegression.md deleted file mode 100644 index 2da17fa6d..000000000 --- a/_tutorials/7_PoissonRegression.md +++ /dev/null @@ -1,487 +0,0 @@ ---- -title: Bayesian Poisson Regression -permalink: /:collection/:name/ ---- - -# Bayesian Poisson Regression -This notebook is ported from the [example notebook](https://docs.pymc.io/notebooks/GLM-poisson-regression.html) of PyMC3 on Poisson Regression. - -[Poisson Regression](https://en.wikipedia.org/wiki/Poisson_regression) is a technique commonly used to model count data. Some of the applications include predicting the number of people defaulting on their loans or the number of cars running on a highway on a given day. This example describes a method to implement the Bayesian version of this technique using Turing. - -We will generate the dataset that we will be working on which describes the relationship between number of times a person sneezes during the day with his alcohol consumption and medicinal intake. - -We start by importing the required libraries. - - -```julia -#Import Turing, Distributions and DataFrames -using Turing, Distributions, DataFrames, Distributed - -# Import MCMCChain, Plots, and StatsPlots for visualizations and diagnostics. -using MCMCChains, Plots, StatsPlots - -# Set a seed for reproducibility. -using Random -Random.seed!(12); - -# Turn off progress monitor. -Turing.turnprogress(false) -``` - - ┌ Info: [Turing]: progress logging is disabled globally - └ @ Turing /home/cameron/.julia/packages/Turing/cReBm/src/Turing.jl:22 - - - - - - false - - - -# Generating data -We start off by creating a toy dataset. We take the case of a person who takes medicine to prevent excessive sneezing. Alcohol consumption increases the rate of sneezing for that person. Thus, the two factors affecting the number of sneezes in a given day are alcohol consumption and whether the person has taken his medicine. Both these variable are taken as boolean valued while the number of sneezes will be a count valued variable. We also take into consideration that the interaction between the two boolean variables will affect the number of sneezes - -5 random rows are printed from the generated data to get a gist of the data generated. - - -```julia -theta_noalcohol_meds = 1 # no alcohol, took medicine -theta_alcohol_meds = 3 # alcohol, took medicine -theta_noalcohol_nomeds = 6 # no alcohol, no medicine -theta_alcohol_nomeds = 36 # alcohol, no medicine - -# no of samples for each of the above cases -q = 100 - -#Generate data from different Poisson distributions -noalcohol_meds = Poisson(theta_noalcohol_meds) -alcohol_meds = Poisson(theta_alcohol_meds) -noalcohol_nomeds = Poisson(theta_noalcohol_nomeds) -alcohol_nomeds = Poisson(theta_alcohol_nomeds) - -nsneeze_data = vcat(rand(noalcohol_meds, q), rand(alcohol_meds, q), rand(noalcohol_nomeds, q), rand(alcohol_nomeds, q) ) -alcohol_data = vcat(zeros(q), ones(q), zeros(q), ones(q) ) -meds_data = vcat(zeros(q), zeros(q), ones(q), ones(q) ) - -df = DataFrame(nsneeze = nsneeze_data, alcohol_taken = alcohol_data, nomeds_taken = meds_data, product_alcohol_meds = meds_data.*alcohol_data) -df[sample(1:nrow(df), 5, replace = false), :] -``` - - - - -

5 rows × 4 columns

nsneezealcohol_takennomeds_takenproduct_alcohol_meds
Int64Float64Float64Float64
150.00.00.0
251.00.00.0
381.00.00.0
410.00.00.0
5381.01.01.0
- - - -# Visualisation of the dataset -We plot the distribution of the number of sneezes for the 4 different cases taken above. As expected, the person sneezes the most when he has taken alcohol and not taken his medicine. He sneezes the least when he doesn't consume alcohol and takes his medicine. - - -```julia -#Data Plotting - -p1 = Plots.histogram(df[(df[:,:alcohol_taken] .== 0) .& (df[:,:nomeds_taken] .== 0), 1], title = "no_alcohol+meds") -p2 = Plots.histogram((df[(df[:,:alcohol_taken] .== 1) .& (df[:,:nomeds_taken] .== 0), 1]), title = "alcohol+meds") -p3 = Plots.histogram((df[(df[:,:alcohol_taken] .== 0) .& (df[:,:nomeds_taken] .== 1), 1]), title = "no_alcohol+no_meds") -p4 = Plots.histogram((df[(df[:,:alcohol_taken] .== 1) .& (df[:,:nomeds_taken] .== 1), 1]), title = "alcohol+no_meds") -plot(p1, p2, p3, p4, layout = (2, 2), legend = false) -``` - - - - -![svg](../7_PoissonRegression_files/7_PoissonRegression_5_0.svg) - - - -We must convert our `DataFrame` data into the `Matrix` form as the manipulations that we are about are designed to work with `Matrix` data. We also separate the features from the labels which will be later used by the Turing sampler to generate samples from the posterior. - - -```julia -# Convert the DataFrame object to matrices. -data = Matrix(df[:,[:alcohol_taken, :nomeds_taken, :product_alcohol_meds]]) -data_labels = df[:,:nsneeze] -data -``` - - - - - 400×3 Array{Float64,2}: - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - 0.0 0.0 0.0 - ⋮ - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - 1.0 1.0 1.0 - - - -We must recenter our data about 0 to help the Turing sampler in initialising the parameter estimates. So, normalising the data in each column by subtracting the mean and dividing by the standard deviation: - - -```julia -# # Rescale our matrices. -data = (data .- mean(data, dims=1)) ./ std(data, dims=1) -``` - - - - - 400×3 Array{Float64,2}: - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - -0.998749 -0.998749 -0.576628 - ⋮ - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - 0.998749 0.998749 1.72988 - - - -# Declaring the Model: Poisson Regression -Our model, `poisson_regression` takes four arguments: - -- `x` is our set of independent variables; -- `y` is the element we want to predict; -- `n` is the number of observations we have; and -- `σ²` is the standard deviation we want to assume for our priors. - -Within the model, we create four coefficients (`b0`, `b1`, `b2`, and `b3`) and assign a prior of normally distributed with means of zero and standard deviations of `σ²`. We want to find values of these four coefficients to predict any given `y`. - -Intuitively, we can think of the coefficients as: - -- `b1` is the coefficient which represents the effect of taking alcohol on the number of sneezes; -- `b2` is the coefficient which represents the effect of taking in no medicines on the number of sneezes; -- `b3` is the coefficient which represents the effect of interaction between taking alcohol and no medicine on the number of sneezes; - -The `for` block creates a variable `theta` which is the weighted combination of the input features. We have defined the priors on these weights above. We then observe the likelihood of calculating `theta` given the actual label, `y[i]`. - - -```julia -# Bayesian poisson regression (LR) -@model poisson_regression(x, y, n, σ²) = begin - b0 ~ Normal(0, σ²) - b1 ~ Normal(0, σ²) - b2 ~ Normal(0, σ²) - b3 ~ Normal(0, σ²) - for i = 1:n - theta = b0 + b1*x[i, 1] + b2*x[i,2] + b3*x[i,3] - y[i] ~ Poisson(exp(theta)) - end -end; -``` - -# Sampling from the posterior -We use the `NUTS` sampler to sample values from the posterior. We run multiple chains using the `mapreduce` function to nullify the effect of a problematic chain. We then use the Gelman, Rubin, and Brooks Diagnostic to check the convergence of these multiple chains. - - -```julia -# Retrieve the number of observations. -n, _ = size(data) - -# Sample using NUTS. - -num_chains = 4 -chain = mapreduce( - c -> sample(poisson_regression(data, data_labels, n, 10), NUTS(200, 0.65), 2500, discard_adapt=false), - chainscat, - 1:num_chains); -``` - - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Info: Found initial step size - │ ϵ = 2.384185791015625e-8 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/cReBm/src/inference/hmc.jl:556 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Info: Found initial step size - │ ϵ = 0.00078125 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/cReBm/src/inference/hmc.jl:556 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - ┌ Info: Found initial step size - │ ϵ = 0.000390625 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/cReBm/src/inference/hmc.jl:556 - ┌ Info: Found initial step size - │ ϵ = 0.05 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/cReBm/src/inference/hmc.jl:556 - ┌ Warning: The current proposal will be rejected due to numerical error(s). - │ isfinite.((θ, r, ℓπ, ℓκ)) = (true, false, false, false) - └ @ AdvancedHMC /home/cameron/.julia/packages/AdvancedHMC/WJCQA/src/hamiltonian.jl:47 - - -# Viewing the Diagnostics -We use the Gelman, Rubin, and Brooks Diagnostic to check whether our chains have converged. Note that we require multiple chains to use this diagnostic which analyses the difference between these multiple chains. - -We expect the chains to have converged. This is because we have taken sufficient number of iterations (1500) for the NUTS sampler. However, in case the test fails, then we will have to take a larger number of iterations, resulting in longer computation time. - - -```julia -gelmandiag(chain) -``` - - - - - Gelman, Rubin, and Brooks Diagnostic - parameters PSRF 97.5% - ────────── ────── ────── - b0 1.1861 1.3924 - b1 1.1307 1.2582 - b2 1.1350 1.2865 - b3 1.0660 1.1118 - - - - -From the above diagnostic, we can conclude that the chains have converged because the PSRF values of the coefficients are close to 1. - -So, we have obtained the posterior distributions of the parameters. We transform the coefficients and recover theta values by taking the exponent of the meaned values of the coefficients `b0`, `b1`, `b2` and `b3`. We take the exponent of the means to get a better comparison of the relative values of the coefficients. We then compare this with the intuitive meaning that was described earlier. - - -```julia -# Taking the first chain -c1 = chain[:,:,1] - -# Calculating the exponentiated means -b0_exp = exp(mean(c1[:b0].value)) -b1_exp = exp(mean(c1[:b1].value)) -b2_exp = exp(mean(c1[:b2].value)) -b3_exp = exp(mean(c1[:b3].value)) - -print("The exponent of the meaned values of the weights (or coefficients are): \n") -print("b0: ", b0_exp, " \n", "b1: ", b1_exp, " \n", "b2: ", b2_exp, " \n", "b3: ", b3_exp, " \n") -print("The posterior distributions obtained after sampling can be visualised as :\n") -``` - - The exponent of the meaned values of the weights (or coefficients are): - b0: 5.116678482496325 - b1: 1.8791946940293356 - b2: 2.5245646467859904 - b3: 1.3005130214177183 - The posterior distributions obtained after sampling can be visualised as : - - - Visualising the posterior by plotting it: - - -```julia -plot(chain) -``` - - - - -![svg](../7_PoissonRegression_files/7_PoissonRegression_19_0.svg) - - - -# Interpreting the Obtained Mean Values -The exponentiated mean of the coefficient `b1` is roughly half of that of `b2`. This makes sense because in the data that we generated, the number of sneezes was more sensitive to the medicinal intake as compared to the alcohol consumption. We also get a weaker dependence on the interaction between the alcohol consumption and the medicinal intake as can be seen from the value of `b3`. - -# Removing the Warmup Samples - -As can be seen from the plots above, the parameters converge to their final distributions after a few iterations. These initial values during the warmup phase increase the standard deviations of the parameters and are not required after we get the desired distributions. Thus, we remove these warmup values and once again view the diagnostics. - -To remove these warmup values, we take all values except the first 200. This is because we set the second parameter of the NUTS sampler (which is the number of adaptations) to be equal to 200. `describe(chain)` is used to view the standard deviations in the estimates of the parameters. It also gives other useful information such as the means and the quantiles. - - -```julia -# Note the standard deviation before removing the warmup samples -describe(chain) -``` - - - - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ─────── ────── - b0 1.2639 2.1637 0.0216 0.2114 42.6654 1.0565 - b1 0.7091 0.8433 0.0084 0.0728 41.7860 1.0620 - b2 1.1998 1.7572 0.0176 0.1676 42.5718 1.0675 - b3 0.2357 0.7392 0.0074 0.0596 91.3888 1.0240 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ─────── ────── ────── ────── ────── - b0 -4.7815 1.6189 1.6409 1.6624 1.7026 - b1 0.4366 0.5151 0.5548 0.5986 3.7771 - b2 0.7707 0.8461 0.8848 0.9259 8.4861 - b3 -1.7651 0.2497 0.2882 0.3275 0.4136 - - - - - -```julia -# Removing the first 200 values of the chains. -chains_new = chain[201:2500,:,:] -describe(chains_new) -``` - - - - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────── ────── ────── ──────── ────── ─────── ────── - b0 1.6378 0.0823 0.0009 0.0055 46.6518 1.0182 - b1 0.5639 0.1729 0.0018 0.0117 45.5782 1.0196 - b2 0.8932 0.1727 0.0018 0.0118 45.0961 1.0195 - b3 0.2798 0.1544 0.0016 0.0104 46.0058 1.0195 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────── ────── ────── ────── ────── ────── - b0 1.5791 1.6226 1.6427 1.6637 1.7024 - b1 0.4413 0.5142 0.5516 0.5919 0.6726 - b2 0.7764 0.8448 0.8819 0.9187 0.9973 - b3 0.1785 0.2544 0.2893 0.3266 0.3942 - - - - -Visualising the new posterior by plotting it: - - -```julia -plot(chains_new) -``` - - - - -![svg](../7_PoissonRegression_files/7_PoissonRegression_25_0.svg) - - - -As can be seen from the numeric values and the plots above, the standard deviation values have decreased and all the plotted values are from the estimated posteriors. The exponentiated mean values, with the warmup samples removed, have not changed by much and they are still in accordance with their intuitive meanings as described earlier. diff --git a/_tutorials/7_PoissonRegression_files/7_PoissonRegression_19_0.svg b/_tutorials/7_PoissonRegression_files/7_PoissonRegression_19_0.svg deleted file mode 100644 index 22ffd816f..000000000 --- a/_tutorials/7_PoissonRegression_files/7_PoissonRegression_19_0.svg +++ /dev/null @@ -1,5489 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - -2000 - - -2500 - - --20 - - --15 - - --10 - - --5 - - -0 - - -5 - - -b0 - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --20 - - --15 - - --10 - - --5 - - -0 - - -5 - - -0 - - -2 - - -4 - - -6 - - -8 - - -10 - - -10 - - -b0 - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - -2000 - - -2500 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - -7.5 - - -10.0 - - -b1 - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - -7.5 - - -10.0 - - -0 - - -1 - - -2 - - -3 - - -4 - - -5 - - -6 - - -b1 - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - -2000 - - -2500 - - --10 - - --5 - - -0 - - -5 - - -10 - - -15 - - -b2 - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --10 - - --5 - - -0 - - -5 - - -10 - - -15 - - -0 - - -2 - - -4 - - -6 - - -b2 - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - -2000 - - -2500 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - -7.5 - - -b3 - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - -7.5 - - -0 - - -1 - - -2 - - -3 - - -4 - - -5 - - -6 - - -b3 - - -Sample value - - -Density - - - - - - diff --git a/_tutorials/7_PoissonRegression_files/7_PoissonRegression_25_0.svg b/_tutorials/7_PoissonRegression_files/7_PoissonRegression_25_0.svg deleted file mode 100644 index e31de4bc9..000000000 --- a/_tutorials/7_PoissonRegression_files/7_PoissonRegression_25_0.svg +++ /dev/null @@ -1,5106 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -500 - - -1000 - - -1500 - - -2000 - - -2500 - - -0.25 - - -0.50 - - -0.75 - - -1.00 - - -1.25 - - -1.50 - - -1.75 - - -b0 - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.25 - - -0.50 - - -0.75 - - -1.00 - - -1.25 - - -1.50 - - -1.75 - - -0 - - -2 - - -4 - - -6 - - -8 - - -10 - - -10 - - -b0 - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -500 - - -1000 - - -1500 - - -2000 - - -2500 - - -0.5 - - -1.0 - - -1.5 - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -b1 - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.5 - - -1.0 - - -1.5 - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -0 - - -2 - - -4 - - -6 - - -b1 - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -500 - - -1000 - - -1500 - - -2000 - - -2500 - - -1 - - -2 - - -3 - - -4 - - -b2 - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 - - -2 - - -3 - - -4 - - -0 - - -2 - - -4 - - -6 - - -b2 - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -500 - - -1000 - - -1500 - - -2000 - - -2500 - - --2.5 - - --2.0 - - --1.5 - - --1.0 - - --0.5 - - -0.0 - - -0.5 - - -b3 - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2.5 - - --2.0 - - --1.5 - - --1.0 - - --0.5 - - -0.0 - - -0.5 - - -0 - - -2 - - -4 - - -6 - - -8 - - -b3 - - -Sample value - - -Density - - - - - - diff --git a/_tutorials/7_PoissonRegression_files/7_PoissonRegression_5_0.svg b/_tutorials/7_PoissonRegression_files/7_PoissonRegression_5_0.svg deleted file mode 100644 index c98293a70..000000000 --- a/_tutorials/7_PoissonRegression_files/7_PoissonRegression_5_0.svg +++ /dev/null @@ -1,624 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -1 - - -2 - - -3 - - -4 - - -5 - - -6 - - -0 - - -10 - - -20 - - -30 - - -40 - - -no_alcohol+meds - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -2 - - -4 - - -6 - - -8 - - -0 - - -5 - - -10 - - -15 - - -20 - - -alcohol+meds - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -5 - - -10 - - -15 - - -0 - - -10 - - -20 - - -30 - - -no_alcohol+no_meds - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -20 - - -30 - - -40 - - -50 - - -0 - - -5 - - -10 - - -15 - - -20 - - -25 - - -alcohol+no_meds - - - - - - - - - - - - - - - - diff --git a/_tutorials/8_MultinomialLogisticRegression.md b/_tutorials/8_MultinomialLogisticRegression.md deleted file mode 100644 index d2efa5d94..000000000 --- a/_tutorials/8_MultinomialLogisticRegression.md +++ /dev/null @@ -1,378 +0,0 @@ ---- -title: Bayesian Multinomial Logistic Regression -permalink: /:collection/:name/ ---- - -# Bayesian Multinomial Logistic Regression -[Multinomial logistic regression](https://en.wikipedia.org/wiki/Multinomial_logistic_regression) is an extension of logistic regression. Logistic regression is used to model problems in which there are exactly two possible discrete outcomes. Multinomial logistic regression is used to model problems in which there are two or more possible discrete outcomes. - -In our example, we'll be using the iris dataset. The goal of the iris multiclass problem is to predict the species of a flower given measurements (in centimeters) of sepal length and width and petal length and width. There are three possible species: Iris setosa, Iris versicolor, and Iris virginica. - -To start, let's import all the libraries we'll need. - - -```julia -# Import Turing and Distributions. -using Turing, Distributions - -# Import RDatasets. -using RDatasets - -# Import MCMCChains, Plots, and StatsPlots for visualizations and diagnostics. -using MCMCChains, Plots, StatsPlots - -# We need a softmax function, which is provided by NNlin. -using NNlib: softmax - -# Set a seed for reproducibility. -using Random -Random.seed!(0); -``` - -## Data Cleaning & Set Up - -Now we're going to import our dataset. Twenty rows of the dataset are shown below so you can get a good feel for what kind of data we have. - - -```julia -# Import the "iris" dataset. -data = RDatasets.dataset("datasets", "iris"); - -# Randomly shuffle the rows of the dataset -num_rows = size(data, 1) -data = data[Random.shuffle(1:num_rows), :] - -# Show twenty rows -first(data, 20) -``` - - - - -

20 rows × 5 columns

SepalLengthSepalWidthPetalLengthPetalWidthSpecies
Float64Float64Float64Float64Categorical…
16.93.25.72.3virginica
25.82.75.11.9virginica
36.62.94.61.3versicolor
46.32.55.01.9virginica
55.02.03.51.0versicolor
65.84.01.20.2setosa
76.73.14.71.5versicolor
85.72.84.51.3versicolor
96.32.95.61.8virginica
105.63.04.11.3versicolor
115.62.74.21.3versicolor
125.13.41.50.2setosa
136.73.35.72.1virginica
145.82.64.01.2versicolor
156.42.94.31.3versicolor
164.83.01.40.1setosa
176.33.45.62.4virginica
184.92.54.51.7virginica
194.83.41.60.2setosa
205.02.33.31.0versicolor
- - - -In this data set, the outcome `Species` is currently coded as a string. We need to convert the `Species` into 1s and 0s. - -We will create three new columns: `Species_setosa`, `Species_versicolor` and `Species_virginica`. - -- If a row has `setosa` as the species, then it will have `Species_setosa = 1`, `Species_versicolor = 0`, and `Species_virginica = 0`. -- If a row has `versicolor` as the species, then it will have `Species_setosa = 0`, `Species_versicolor = 1`, and `Species_virginica = 0`. -- If a row has `virginica` as the species, then it will have `Species_setosa = 0`, `Species_versicolor = 0`, and `Species_virginica = 1`. - - -```julia -# Recode the `Species` column -data[!, :Species_setosa] = [r.Species == "setosa" ? 1.0 : 0.0 for r in eachrow(data)] -data[!, :Species_versicolor] = [r.Species == "versicolor" ? 1.0 : 0.0 for r in eachrow(data)] -data[!, :Species_virginica] = [r.Species == "virginica" ? 1.0 : 0.0 for r in eachrow(data)] - -# Show twenty rows of the new species columns -first(data[!, [:Species, :Species_setosa, :Species_versicolor, :Species_virginica]], 20) -``` - - - - -

20 rows × 4 columns

SpeciesSpecies_setosaSpecies_versicolorSpecies_virginica
Categorical…Float64Float64Float64
1virginica0.00.01.0
2virginica0.00.01.0
3versicolor0.01.00.0
4virginica0.00.01.0
5versicolor0.01.00.0
6setosa1.00.00.0
7versicolor0.01.00.0
8versicolor0.01.00.0
9virginica0.00.01.0
10versicolor0.01.00.0
11versicolor0.01.00.0
12setosa1.00.00.0
13virginica0.00.01.0
14versicolor0.01.00.0
15versicolor0.01.00.0
16setosa1.00.00.0
17virginica0.00.01.0
18virginica0.00.01.0
19setosa1.00.00.0
20versicolor0.01.00.0
- - - -After we've done that tidying, it's time to split our dataset into training and testing sets, and separate the labels from the data. We separate our data into two halves, `train` and `test`. - -We must rescale our feature variables so that they are centered around zero by subtracting each column by the mean and dividing it by the standard deviation. Without this step, Turing's sampler will have a hard time finding a place to start searching for parameter estimates. - - -```julia -# Function to split samples. -function split_data(df, at) - (r, _) = size(df) - index = Int(round(r * at)) - train = df[1:index, :] - test = df[(index+1):end, :] - return train, test -end - -# Rescale our feature variables. -data.SepalLength = (data.SepalLength .- mean(data.SepalLength)) ./ std(data.SepalLength) -data.SepalWidth = (data.SepalWidth .- mean(data.SepalWidth)) ./ std(data.SepalWidth) -data.PetalLength = (data.PetalLength .- mean(data.PetalLength)) ./ std(data.PetalLength) -data.PetalWidth = (data.PetalWidth .- mean(data.PetalWidth)) ./ std(data.PetalWidth) - -# Split our dataset 50/50 into training/test sets. -train, test = split_data(data, 0.50); - -label_names = [:Species_setosa, :Species_versicolor, :Species_virginica] -feature_names = [:SepalLength, :SepalWidth, :PetalLength, :PetalWidth] - -# Create our labels. These are the values we are trying to predict. -train_labels = train[:, label_names] -test_labels = test[:, label_names] - -# Create our features. These are our predictors. -train_features = train[:, feature_names]; -test_features = test[:, feature_names]; -``` - -Our `train` and `test` matrices are still in the `DataFrame` format, which tends not to play too well with the kind of manipulations we're about to do, so we convert them into `Matrix` objects. - - -```julia -# Convert the DataFrame objects to matrices. -train_labels = Matrix(train_labels); -test_labels = Matrix(test_labels); - -train_features = Matrix(train_features); -test_features = Matrix(test_features); -``` - -## Model Declaration -Finally, we can define our model. - -`logistic_regression` takes four arguments: - -- `x` is our set of independent variables; -- `y` is the element we want to predict; -- `n` is the number of observations we have; and -- `σ` is the standard deviation we want to assume for our priors. - -We need to create our coefficients. To do so, we first need to select one of the species as the baseline species. The selection of the baseline class does not matter. Then we create our coefficients against that baseline. - -Let us select `"setosa"` as the baseline. We create ten coefficients (`intercept_versicolor`, `intercept_virginica`, `SepalLength_versicolor`, `SepalLength_virginica`, `SepalWidth_versicolor`, `SepalWidth_virginica`, `PetalLength_versicolor`, `PetalLength_virginica`, `PetalWidth_versicolor`, and `PetalWidth_virginica`) and assign a prior of normally distributed with means of zero and standard deviations of `σ`. We want to find values of these ten coefficients to predict any given `y`. - -The `for` block creates a variable `v` which is the softmax function. We then observe the liklihood of calculating `v` given the actual label, `y[i]`. - - -```julia -# Bayesian multinomial logistic regression -@model logistic_regression(x, y, n, σ) = begin - intercept_versicolor ~ Normal(0, σ) - intercept_virginica ~ Normal(0, σ) - - SepalLength_versicolor ~ Normal(0, σ) - SepalLength_virginica ~ Normal(0, σ) - - SepalWidth_versicolor ~ Normal(0, σ) - SepalWidth_virginica ~ Normal(0, σ) - - PetalLength_versicolor ~ Normal(0, σ) - PetalLength_virginica ~ Normal(0, σ) - - PetalWidth_versicolor ~ Normal(0, σ) - PetalWidth_virginica ~ Normal(0, σ) - - - for i = 1:n - v = softmax([0, # this 0 corresponds to the base category `setosa` - intercept_versicolor + SepalLength_versicolor*x[i, 1] + - SepalWidth_versicolor*x[i, 1] + - PetalLength_versicolor*x[i, 2] + - PetalWidth_versicolor*x[i, 2], - intercept_virginica + SepalLength_virginica*x[i, 3] + - SepalWidth_virginica*x[i, 3] + - PetalLength_virginica*x[i, 4] + - PetalWidth_virginica*x[i, 4]]) - y[i, :] ~ Multinomial(1, v) - end -end; -``` - -## Sampling - -Now we can run our sampler. This time we'll use [`HMC`](http://turing.ml/docs/library/#Turing.HMC) to sample from our posterior. - - -```julia -# Retrieve the number of observations. -n, _ = size(train_features) - -# Sample using HMC. -chain = mapreduce(c -> sample(logistic_regression(train_features, train_labels, n, 1), HMC(0.05, 10), 1500), - chainscat, - 1:3 -) - -describe(chain) -``` - - Sampling: 100%|█████████████████████████████████████████| Time: 0:00:05 - Sampling: 100%|█████████████████████████████████████████| Time: 0:00:04 - Sampling: 100%|█████████████████████████████████████████| Time: 0:00:04 - - - - - - 2-element Array{ChainDataFrame,1} - - Summary Statistics - parameters mean std naive_se mcse ess r_hat - ────────────────────── ─────── ────── ──────── ────── ───────── ────── - PetalLength_versicolor -0.7982 0.7341 0.0109 0.0353 330.7338 1.0022 - PetalLength_virginica 1.7376 0.8532 0.0127 0.0424 415.6667 1.0023 - PetalWidth_versicolor -0.7018 0.7335 0.0109 0.0339 355.4789 1.0017 - PetalWidth_virginica 1.6843 0.8452 0.0126 0.0382 447.2635 1.0089 - SepalLength_versicolor 0.8642 0.7315 0.0109 0.0357 370.1195 1.0052 - SepalLength_virginica 1.5303 0.8641 0.0129 0.0452 321.9949 1.0078 - SepalWidth_versicolor 0.8227 0.7506 0.0112 0.0363 364.8514 1.0036 - SepalWidth_virginica 1.5765 0.8516 0.0127 0.0468 405.3356 1.0078 - intercept_versicolor 1.0275 0.4539 0.0068 0.0156 1004.6000 1.0029 - intercept_virginica -0.9449 0.6155 0.0092 0.0246 700.5740 1.0033 - - Quantiles - parameters 2.5% 25.0% 50.0% 75.0% 97.5% - ────────────────────── ─────── ─────── ─────── ─────── ────── - PetalLength_versicolor -2.2429 -1.2926 -0.7839 -0.2936 0.5790 - PetalLength_virginica 0.0566 1.1492 1.7495 2.3157 3.4072 - PetalWidth_versicolor -2.1443 -1.2000 -0.7049 -0.2173 0.7632 - PetalWidth_virginica -0.0565 1.1274 1.6940 2.2695 3.2582 - SepalLength_versicolor -0.5667 0.3953 0.8762 1.3547 2.2408 - SepalLength_virginica -0.1067 0.9405 1.5259 2.0997 3.2549 - SepalWidth_versicolor -0.5795 0.3072 0.8086 1.3063 2.3417 - SepalWidth_virginica -0.1364 1.0034 1.5684 2.1657 3.2340 - intercept_versicolor 0.1491 0.7267 1.0235 1.3287 1.9327 - intercept_virginica -2.1602 -1.3516 -0.9434 -0.5233 0.2201 - - - - -Since we ran multiple chains, we may as well do a spot check to make sure each chain converges around similar points. - - -```julia -plot(chain) -``` - - - - -![svg](../8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_15_0.svg) - - - -Looks good! - -We can also use the `corner` function from MCMCChains to show the distributions of the various parameters of our multinomial logistic regression. The corner function requires MCMCChains and StatsPlots. - - -```julia -corner(chain, [:SepalLength_versicolor, :SepalWidth_versicolor, :PetalLength_versicolor, :PetalWidth_versicolor]) -``` - - - - -![svg](../8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_17_0.svg) - - - - -```julia -corner(chain, [:SepalLength_versicolor, :SepalWidth_versicolor, :PetalLength_versicolor, :PetalWidth_versicolor]) -``` - - - - -![svg](../8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_18_0.svg) - - - -Fortunately the corner plots appear to demonstrate unimodal distributions for each of our parameters, so it should be straightforward to take the means of each parameter's sampled values to estimate our model to make predictions. - -## Making Predictions -How do we test how well the model actually predicts whether someone is likely to default? We need to build a prediction function that takes the `test` object we made earlier and runs it through the average parameter calculated during sampling. - -The `prediction` function below takes a `Matrix` and a `Chain` object. It takes the mean of each parameter's sampled values and re-runs the softmax function using those mean values for every element in the test set. - - -```julia -function prediction(x::Matrix, chain) - # Pull the means from each parameter's sampled values in the chain. - intercept_versicolor = mean(chain[:intercept_versicolor].value) - intercept_virginica = mean(chain[:intercept_virginica].value) - SepalLength_versicolor = mean(chain[:SepalLength_versicolor].value) - SepalLength_virginica = mean(chain[:SepalLength_virginica].value) - SepalWidth_versicolor = mean(chain[:SepalWidth_versicolor].value) - SepalWidth_virginica = mean(chain[:SepalWidth_virginica].value) - PetalLength_versicolor = mean(chain[:PetalLength_versicolor].value) - PetalLength_virginica = mean(chain[:PetalLength_virginica].value) - PetalWidth_versicolor = mean(chain[:PetalWidth_versicolor].value) - PetalWidth_virginica = mean(chain[:PetalWidth_virginica].value) - - # Retrieve the number of rows. - n, _ = size(x) - - # Generate a vector to store our predictions. - v = Vector{String}(undef, n) - - # Calculate the softmax function for each element in the test set. - for i in 1:n - num = softmax([0, # this 0 corresponds to the base category `setosa` - intercept_versicolor + SepalLength_versicolor*x[i, 1] + - SepalWidth_versicolor*x[i, 1] + - PetalLength_versicolor*x[i, 2] + - PetalWidth_versicolor*x[i, 2], - intercept_virginica + SepalLength_virginica*x[i, 3] + - SepalWidth_virginica*x[i, 3] + - PetalLength_virginica*x[i, 4] + - PetalWidth_virginica*x[i, 4]]) - c = argmax(num) # we pick the class with the highest probability - if c == 1 - v[i] = "setosa" - elseif c == 2 - v[i] = "versicolor" - else # c == 3 - @assert c == 3 - v[i] = "virginica" - end - end - return v -end; -``` - -Let's see how we did! We run the test matrix through the prediction function, and compute the accuracy for our prediction. - - -```julia -# Make the predictions. -predictions = prediction(test_features, chain) - -# Calculate accuracy for our test set. -mean(predictions .== test[!, :Species]) -``` - - - - - 0.8933333333333333 - - - -Perhaps more important is to see the accuracy per class. - - -```julia -setosa_rows = test[!, :Species] .== "setosa" -versicolor_rows = test[!, :Species] .== "versicolor" -virginica_rows = test[!, :Species] .== "virginica" - -println("Number of setosa: $$(sum(setosa_rows))") -println("Number of versicolor: $$(sum(versicolor_rows))") -println("Number of virginica: $$(sum(virginica_rows))") - -println("Percentage of setosa predicted correctly: $$(mean(predictions[setosa_rows] .== test[setosa_rows, :Species]))") -println("Percentage of versicolor predicted correctly: $$(mean(predictions[versicolor_rows] .== test[versicolor_rows, :Species]))") -println("Percentage of virginica predicted correctly: $$(mean(predictions[virginica_rows] .== test[virginica_rows, :Species]))") -``` - - Number of setosa: 32 - Number of versicolor: 25 - Number of virginica: 18 - Percentage of setosa predicted correctly: 0.96875 - Percentage of versicolor predicted correctly: 0.76 - Percentage of virginica predicted correctly: 0.9444444444444444 - - -This tutorial has demonstrated how to use Turing to perform Bayesian multinomial logistic regression. diff --git a/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_15_0.svg b/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_15_0.svg deleted file mode 100644 index 196a424e9..000000000 --- a/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_15_0.svg +++ /dev/null @@ -1,7628 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --3 - - --2 - - --1 - - -0 - - -1 - - -PetalLength_versicolor - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --3 - - --2 - - --1 - - -0 - - -1 - - -2 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -PetalLength_versicolor - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -PetalLength_virginica - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - -0 - - -2 - - -4 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -PetalLength_virginica - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --3 - - --2 - - --1 - - -0 - - -1 - - -2 - - -PetalWidth_versicolor - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --3 - - --2 - - --1 - - -0 - - -1 - - -2 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -PetalWidth_versicolor - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -PetalWidth_virginica - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - -0 - - -2 - - -4 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -PetalWidth_virginica - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --2 - - --1 - - -0 - - -1 - - -2 - - -3 - - -SepalLength_versicolor - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - -0 - - -2 - - -4 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -SepalLength_versicolor - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -SepalLength_virginica - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - -0 - - -2 - - -4 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -SepalLength_virginica - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -SepalWidth_versicolor - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -SepalWidth_versicolor - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -SepalWidth_virginica - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - -0 - - -2 - - -4 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -SepalWidth_virginica - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --0.5 - - -0.0 - - -0.5 - - -1.0 - - -1.5 - - -2.0 - - -2.5 - - -intercept_versicolor - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --1 - - -0 - - -1 - - -2 - - -3 - - -0.0 - - -0.2 - - -0.4 - - -0.6 - - -0.8 - - -intercept_versicolor - - -Sample value - - -Density - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -500 - - -1000 - - -1500 - - --3 - - --2 - - --1 - - -0 - - -1 - - -intercept_virginica - - -Iteration - - -Sample value - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --3 - - --2 - - --1 - - -0 - - -1 - - -0.0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -intercept_virginica - - -Sample value - - -Density - - - - - diff --git a/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_17_0.svg b/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_17_0.svg deleted file mode 100644 index 53a3d00bc..000000000 --- a/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_17_0.svg +++ /dev/null @@ -1,28774 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --3 - - --2 - - --1 - - -0 - - -1 - - -2 - - -SepalWidth_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - --1 - - -0 - - -1 - - -2 - - -3 - - -PetalLength_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --3 - - --2 - - --1 - - -0 - - -1 - - --2 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -SepalLength_versicolor - - -PetalWidth_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --3 - - --2 - - --1 - - -0 - - -1 - - -2 - - -SepalWidth_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - --1 - - -0 - - -1 - - -2 - - -3 - - -PetalLength_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_18_0.svg b/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_18_0.svg deleted file mode 100644 index 74c27d028..000000000 --- a/_tutorials/8_MultinomialLogisticRegression_files/8_MultinomialLogisticRegression_18_0.svg +++ /dev/null @@ -1,28774 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --3 - - --2 - - --1 - - -0 - - -1 - - -2 - - -SepalWidth_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - --1 - - -0 - - -1 - - -2 - - -3 - - -PetalLength_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --3 - - --2 - - --1 - - -0 - - -1 - - --2 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - -SepalLength_versicolor - - -PetalWidth_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --3 - - --2 - - --1 - - -0 - - -1 - - -2 - - -SepalWidth_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --2 - - --1 - - -0 - - -1 - - -2 - - -3 - - -PetalLength_versicolor - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/_tutorials/9_VariationalInference.md b/_tutorials/9_VariationalInference.md deleted file mode 100644 index e0af749a1..000000000 --- a/_tutorials/9_VariationalInference.md +++ /dev/null @@ -1,1325 +0,0 @@ ---- -title: Variational inference (VI) in Turing.jl -permalink: /:collection/:name/ ---- - -# Variational inference (VI) in Turing.jl - -In this post we'll have a look at what's know as **variational inference (VI)**, a family of _approximate_ Bayesian inference methods, and how to use it in Turing.jl as an alternative to other approaches such as MCMC. In particular, we will focus on one of the more standard VI methods called **Automatic Differentation Variational Inference (ADVI)**. - -Here we will focus on how to use VI in Turing and not much on the theory underlying VI. If you're interested in understanding the mathematics you can checkout [our write-up](../../docs/for-developers/variational_inference) or any other resource online (there a lot of great ones). - -Using VI in Turing.jl is very straight forward. If `model` denotes a definition of a `Turing.Model`, performing VI is as simple as -```julia -m = model(data...) # instantiate model on the data -q = vi(m, vi_alg) # perform VI on `m` using the VI method `vi_alg`, which returns a `VariationalPosterior` -``` -Thus it's no more work than standard MCMC sampling in Turing. - -To get a bit more into what we can do with `vi`, we'll first have a look at a simple example and then we'll reproduce the [tutorial on Bayesian linear regression](../../tutorials/5-linearregression) using VI instead of MCMC. Finally we'll look at some of the different parameters of `vi` and how you for example can use your own custom variational family. - -## Setup - - -```julia -using Random -using Turing -using Turing: Variational - -Random.seed!(42); -``` - -## Simple example: Normal-Gamma conjugate model - -The Normal-(Inverse)Gamma conjugate model is defined by the following generative process - -\begin{align} - s &\sim \mathrm{InverseGamma}(2, 3) \\ - m &\sim \mathcal{N}(0, s) \\ - x_i &\overset{\text{i.i.d.}}{=} \mathcal{N}(m, s), \quad i = 1, \dots, n -\end{align} - -Recall that *conjugate* refers to the fact that we can obtain a closed-form expression for the posterior. Of course one wouldn't use something like variational inference for a conjugate model, but it's useful as a simple demonstration as we can compare the result to the true posterior. - -First we generate some synthetic data, define the `Turing.Model` and instantiate the model on the data: - - -```julia -# generate data, n = 2000 -x = randn(2000); -``` - - - - -```julia -@model model(x) = begin - s ~ InverseGamma(2, 3) - m ~ Normal(0.0, sqrt(s)) - for i = 1:length(x) - x[i] ~ Normal(m, sqrt(s)) - end -end -``` - - - - - ##model#344 (generic function with 2 methods) - - - - -```julia -# construct model -m = model(x); -``` - -Now we'll produce some samples from the posterior using a MCMC method, which in constrast to VI is guaranteed to converge to the *exact* posterior (as the number of samples go to infinity). - -We'll produce 10 000 samples with 200 steps used for adaptation and a target acceptance rate of 0.65 - -If you don't understand what "adaptation" or "target acceptance rate" refers to, all you really need to know is that `NUTS` is known to be one of the most accurate and efficient samplers (when applicable) while requiring little to no hand-tuning to work well. - - -```julia -samples_nuts = sample(m, NUTS(200, 0.65), 10000); -``` - - ┌ Info: Found initial step size - │ ϵ = 0.025 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/cReBm/src/inference/hmc.jl:556 - Sampling: 100%|█████████████████████████████████████████| Time: 0:00:02 - - -Now let's try VI. The most important function you need to now about to do VI in Turing is `vi`: - - -```julia -print(@doc(Variational.vi)) -``` - - ``` - vi(model, alg::VariationalInference) - vi(model, alg::VariationalInference, q::VariationalPosterior) - vi(model, alg::VariationalInference, getq::Function, θ::AbstractArray) - ``` - - Constructs the variational posterior from the `model` and performs the optimization following the configuration of the given `VariationalInference` instance. - - # Arguments - - * `model`: `Turing.Model` or `Function` z ↦ log p(x, z) where `x` denotes the observations - * `alg`: the VI algorithm used - * `q`: a `VariationalPosterior` for which it is assumed a specialized implementation of the variational objective used exists. - * `getq`: function taking parameters `θ` as input and returns a `VariationalPosterior` - * `θ`: only required if `getq` is used, in which case it is the initial parameters for the variational posterior - - -`vi` takes the `Model` you want to approximate, a `VariationalInference` whose type specifies the method to use and then its fields specify the configuration of the method. - -Additionally, you can pass -- an initial variational posterior `q`, for which we assume there exists a implementation of `update(::typeof(q), θ::AbstractVector)` returning an updated posterior `q` with parameters `θ`. -- a function mapping $$\theta \mapsto q_{\theta}$$ (denoted above `getq`) together with initial parameters `θ`. This provides more flexibility in the types of variational families that we can use, and can sometimes be slightly more convenient for quick and rough work. - -By default, i.e. when calling `vi(m, advi)`, Turing use a *mean-field* approximation with a multivariate normal as the base-distribution. Mean-field refers to the fact that we assume all the latent variables to be *independent*. This the "standard" ADVI approach; see [Automatic Differentiation Variational Inference (2016)](https://arxiv.org/abs/1603.00788) for more. In Turing, one can obtain such a mean-field approximation by calling `Variational.meanfield(model)` for which there exists an internal implementation for `update`: - - -```julia -print(@doc(Variational.meanfield)) -``` - - ``` - meanfield(model::Model) - meanfield(rng::AbstractRNG, model::Model) - ``` - - Creates a mean-field approximation with multivariate normal as underlying distribution. - - -Currently the only implementation of `VariationalInference` available is `ADVI`, which is very convenient and applicable as long as your `Model` is differentiable with respect to the *variational parameters*, that is, the parameters of your variational distribution, e.g. mean and variance in the mean-field approximation. - - -```julia -print(@doc(Variational.ADVI)) -``` - - ```julia - struct ADVI{AD} <: Turing.Variational.VariationalInference{AD} - ``` - - Automatic Differentiation Variational Inference (ADVI) with automatic differentiation backend `AD`. - - # Fields - - * `samples_per_step::Int64` - - Number of samples used to estimate the ELBO in each optimization step. - * `max_iters::Int64` - - Maximum number of gradient steps. - - ``` - ADVI([samples_per_step=1, max_iters=1000]) - ``` - - Create an [`ADVI`](@ref) with the currently enabled automatic differentiation backend `ADBackend()`. - - -To perform VI on the model `m` using 10 samples for gradient estimation and taking 1000 gradient steps is then as simple as: - - -```julia -# ADVI -advi = ADVI(10, 1000) -q = vi(m, advi); -``` - - ┌ Info: [ADVI] Should only be seen once: optimizer created for θ - │ objectid(θ) = 12334556482979097499 - └ @ Turing.Variational /home/cameron/.julia/packages/Turing/cReBm/src/variational/VariationalInference.jl:204 - [ADVI] Optimizing...: 100%|█████████████████████████████████████████| Time: 0:00:03 - - -Unfortunately, for such a small problem Turing's new `NUTS` sampler is *so* efficient now that it's not that much more efficient to use ADVI. So, so very unfortunate... - -With that being said, this is not the case in general. For very complex models we'll later find that `ADVI` produces very reasonable results in a much shorter time than `NUTS`. - -And one significant advantage of using `vi` is that we can sample from the resulting `q` with ease. In fact, the result of the `vi` call is a `TransformedDistribution` from Bijectors.jl, and it implements the Distributions.jl interface for a `Distribution`: - - -```julia -q isa MultivariateDistribution -``` - - - - - true - - - -This means that we can call `rand` to sample from the variational posterior `q` - - -```julia -rand(q) -``` - - - - - 2-element Array{Float64,1}: - 1.0134702063474585 - -0.07429020521027016 - - - -and `logpdf` to compute the log-probability - - -```julia -logpdf(q, rand(q)) -``` - - - - - 4.277478745320889 - - - -Let's check the first and second moments of the data to see how our approximation compares to the point-estimates form the data: - - -```julia -var(x), mean(x) -``` - - - - - (1.021109459575047, -0.028838703049547422) - - - - -```julia -(mean(rand(q, 1000); dims = 2)..., ) -``` - - - - - (1.02716749432684, -0.02510701319723139) - - - -That's pretty close! But we're Bayesian so we're not interested in *just* matching the mean. -Let's instead look the actual density `q`. - -For that we need samples: - - -```julia -samples = rand(q, 10000); -``` - - -```julia -# setup for plotting -using Plots, LaTeXStrings, StatsPlots -pyplot() -``` - - ┌ Info: Precompiling PyPlot [d330b81b-6aea-500a-939a-2ce795aea3ee] - └ @ Base loading.jl:1260 - - - - - - Plots.PyPlotBackend() - - - - -```julia -p1 = histogram(samples[1, :], bins=100, normed=true, alpha=0.2, color = :blue, label = "") -density!(samples[1, :], label = "s (ADVI)", color = :blue, linewidth = 2) -density!(collect(skipmissing(samples_nuts[:s].value)), label = "s (NUTS)", color = :green, linewidth = 2) -vline!([var(x)], label = "s (data)", color = :black) -vline!([mean(samples[1, :])], color = :blue, label ="") - -p2 = histogram(samples[2, :], bins=100, normed=true, alpha=0.2, color = :blue, label = "") -density!(samples[2, :], label = "m (ADVI)", color = :blue, linewidth = 2) -density!(collect(skipmissing(samples_nuts[:m].value)), label = "m (NUTS)", color = :green, linewidth = 2) -vline!([mean(x)], color = :black, label = "m (data)") -vline!([mean(samples[2, :])], color = :blue, label="") - -plot(p1, p2, layout=(2, 1), size=(900, 500)) -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_34_0.png) - - - -For this particular `Model`, we can in fact obtain the posterior of the latent variables in closed form. This allows us to compare both `NUTS` and `ADVI` to the true posterior $$p(s, m \mid \{x_i\}_{i = 1}^n )$$. - -*The code below is just work to get the marginals $$p(s \mid \{x_i\}_{i = 1}^n)$$ and $$p(m \mid \{x_i\}_{i = 1}^n$$ from the posterior obtained using ConjugatePriors.jl. Feel free to skip it.* - - -```julia -# used to compute closed form expression of posterior -using ConjugatePriors - -# closed form computation -# notation mapping has been verified by explicitly computing expressions -# in "Conjugate Bayesian analysis of the Gaussian distribution" by Murphy -μ₀ = 0.0 # => μ -κ₀ = 1.0 # => ν, which scales the precision of the Normal -α₀ = 2.0 # => "shape" -β₀ = 3.0 # => "rate", which is 1 / θ, where θ is "scale" - -# prior -pri = NormalGamma(μ₀, κ₀, α₀, β₀) - -# posterior -post = posterior(pri, Normal, x) - -# marginal distribution of τ = 1 / σ² -# Eq. (90) in "Conjugate Bayesian analysis of the Gaussian distribution" by Murphy -# `scale(post)` = θ -p_τ = Gamma(post.shape, scale(post)) -p_σ²_pdf = z -> pdf(p_τ, 1 / z) # τ => 1 / σ² - -# marginal of μ -# Eq. (91) in "Conjugate Bayesian analysis of the Gaussian distribution" by Murphy -p_μ = TDist(2 * post.shape) - -μₙ = post.mu # μ → μ -κₙ = post.nu # κ → ν -αₙ = post.shape # α → shape -βₙ = post.rate # β → rate - -# numerically more stable but doesn't seem to have effect; issue is probably internal to -# `pdf` which needs to compute ≈ Γ(1000) -p_μ_pdf = z -> exp(logpdf(p_μ, (z - μₙ) * exp(- 0.5 * log(βₙ) + 0.5 * log(αₙ) + 0.5 * log(κₙ)))) - -# posterior plots -p1 = plot(); -histogram!(samples[1, :], bins=100, normed=true, alpha=0.2, color = :blue, label = "") -density!(samples[1, :], label = "s (ADVI)", color = :blue) -density!(collect(skipmissing(samples_nuts[:s].value)), label = "s (NUTS)", color = :green) -vline!([mean(samples[1, :])], linewidth = 1.5, color = :blue, label ="") - -# normalize using Riemann approx. because of (almost certainly) numerical issues -Δ = 0.001 -r = 0.75:0.001:1.50 -norm_const = sum(p_σ²_pdf.(r) .* Δ) -plot!(r, p_σ²_pdf, label = "s (posterior)", color = :red); -vline!([var(x)], label = "s (data)", linewidth = 1.5, color = :black, alpha = 0.7); -xlims!(0.75, 1.35); - -p2 = plot(); -histogram!(samples[2, :], bins=100, normed=true, alpha=0.2, color = :blue, label = "") -density!(samples[2, :], label = "m (ADVI)", color = :blue) -density!(collect(skipmissing(samples_nuts[:m].value)), label = "m (NUTS)", color = :green) -vline!([mean(samples[2, :])], linewidth = 1.5, color = :blue, label="") - - -# normalize using Riemann approx. because of (almost certainly) numerical issues -Δ = 0.0001 -r = -0.1 + mean(x):Δ:0.1 + mean(x) -norm_const = sum(p_μ_pdf.(r) .* Δ) -plot!(r, z -> p_μ_pdf(z) / norm_const, label = "m (posterior)", color = :red); -vline!([mean(x)], label = "m (data)", linewidth = 1.5, color = :black, alpha = 0.7); - -xlims!(-0.25, 0.25); - -p = plot(p1, p2; layout=(2, 1), size=(900, 500)) -``` - - ┌ Info: Precompiling ConjugatePriors [1624bea9-42b1-5fc1-afd3-e96f729c8d6c] - └ @ Base loading.jl:1260 - - - - - -![png](../9_VariationalInference_files/9_VariationalInference_36_1.png) - - - -# Bayesian linear regression example using `ADVI` - -This is simply a duplication of the tutorial [5. Linear regression](../../tutorials/5-linearregression) but now with the addition of an approximate posterior obtained using `ADVI`. - -As we'll see, there is really no additional work required to apply variational inference to a more complex `Model`. - -## Copy-paste from [5. Linear regression](../../tutorials/5-linearregression) - -This section is basically copy-pasting the code from the [linear regression tutorial](../../tutorials/5-linearregression). - - -```julia -Random.seed!(1); -``` - - -```julia -# Import RDatasets. -using RDatasets - -# Hide the progress prompt while sampling. -Turing.turnprogress(true); -``` - - ┌ Info: [Turing]: progress logging is enabled globally - └ @ Turing /home/cameron/.julia/packages/Turing/cReBm/src/Turing.jl:22 - - - -```julia -# Import the "Default" dataset. -data = RDatasets.dataset("datasets", "mtcars"); - -# Show the first six rows of the dataset. -first(data, 6) -``` - - - - -

6 rows × 12 columns (omitted printing of 3 columns)

ModelMPGCylDispHPDRatWTQSecVS
StringFloat64Int64Float64Int64Float64Float64Float64Int64
1Mazda RX421.06160.01103.92.6216.460
2Mazda RX4 Wag21.06160.01103.92.87517.020
3Datsun 71022.84108.0933.852.3218.611
4Hornet 4 Drive21.46258.01103.083.21519.441
5Hornet Sportabout18.78360.01753.153.4417.020
6Valiant18.16225.01052.763.4620.221
- - - - -```julia -# Function to split samples. -function split_data(df, at = 0.70) - r = size(df,1) - index = Int(round(r * at)) - train = df[1:index, :] - test = df[(index+1):end, :] - return train, test -end - -# A handy helper function to rescale our dataset. -function standardize(x) - return (x .- mean(x, dims=1)) ./ std(x, dims=1), x -end - -# Another helper function to unstandardize our datasets. -function unstandardize(x, orig) - return (x .+ mean(orig, dims=1)) .* std(orig, dims=1) -end -``` - - - - - unstandardize (generic function with 1 method) - - - - -```julia -# Remove the model column. -select!(data, Not(:Model)) - -# Standardize our dataset. -(std_data, data_arr) = standardize(Matrix(data)) - -# Split our dataset 70%/30% into training/test sets. -train, test = split_data(std_data, 0.7) - -# Save dataframe versions of our dataset. -train_cut = DataFrame(train, names(data)) -test_cut = DataFrame(test, names(data)) - -# Create our labels. These are the values we are trying to predict. -train_label = train_cut[:, :MPG] -test_label = test_cut[:, :MPG] - -# Get the list of columns to keep. -remove_names = filter(x->!in(x, [:MPG, :Model]), names(data)) - -# Filter the test and train sets. -train = Matrix(train_cut[:,remove_names]); -test = Matrix(test_cut[:,remove_names]); -``` - - -```julia -# Bayesian linear regression. -@model linear_regression(x, y, n_obs, n_vars, ::Type{T}=Vector{Float64}) where {T} = begin - # Set variance prior. - σ₂ ~ truncated(Normal(0,100), 0, Inf) - - # Set intercept prior. - intercept ~ Normal(0, 3) - - # Set the priors on our coefficients. - coefficients ~ MvNormal(zeros(n_vars), 10 * ones(n_vars)) - - # Calculate all the mu terms. - mu = intercept .+ x * coefficients - y ~ MvNormal(mu, σ₂) -end; -``` - - -```julia -n_obs, n_vars = size(train) -m = linear_regression(train, train_label, n_obs, n_vars); -``` - -## Performing VI - -First we define the initial variational distribution, or, equivalently, the family of distributions to consider. We're going to use the same mean-field approximation as Turing will use by default when we call `vi(m, advi)`, which we obtain by calling `Variational.meanfield`. This returns a `TransformedDistribution` with a `TuringDiagMvNormal` as the underlying distribution and the transformation mapping from the reals to the domain of the latent variables. - - -```julia -q0 = Variational.meanfield(m) -typeof(q0) -``` - - - - - Bijectors.TransformedDistribution{DistributionsAD.TuringDiagMvNormal{Array{Float64,1},Array{Float64,1}},Bijectors.Stacked{Tuple{Bijectors.Inverse{Bijectors.TruncatedBijector{Float64},0},Bijectors.Identity{0},Bijectors.Identity{1}},3},Multivariate} - - - - -```julia -advi = ADVI(10, 10_000) -``` - - - - - ADVI{Turing.Core.ForwardDiffAD{40}}(10, 10000) - - - -Turing also provides a couple of different optimizers: -- `TruncatedADAGrad` (default) -- `DecayedADAGrad` -as these are well-suited for problems with high-variance stochastic objectives, which is usually what the ELBO ends up being at different times in our optimization process. - -With that being said, thanks to Requires.jl, if we add a `using Flux` prior to `using Turing` we can also make use of all the optimizers in `Flux`, e.g. `ADAM`, without any additional changes to your code! For example: -```julia -using Flux, Turing -using Turing.Variational - -vi(m, advi; optimizer = Flux.ADAM()) -``` -just works. - -For this problem we'll use the `DecayedADAGrad` from Turing: - - -```julia -opt = Variational.DecayedADAGrad(1e-2, 1.1, 0.9) -``` - - - - - Turing.Variational.DecayedADAGrad(0.01, 1.1, 0.9, IdDict{Any,Any}()) - - - - -```julia -q = vi(m, advi, q0; optimizer = opt) -typeof(q) -``` - - [ADVI] Optimizing...: 100%|█████████████████████████████████████████| Time: 0:00:05 - - - - - - Bijectors.TransformedDistribution{DistributionsAD.TuringDiagMvNormal{Array{Float64,1},Array{Float64,1}},Bijectors.Stacked{Tuple{Bijectors.Inverse{Bijectors.TruncatedBijector{Float64},0},Bijectors.Identity{0},Bijectors.Identity{1}},3},Multivariate} - - - -*Note: as mentioned before, we internally define a `update(q::TransformedDistribution{<:TuringDiagMvNormal}, θ::AbstractVector)` method which takes in the current variational approximation `q` together with new parameters `z` and returns the new variational approximation. This is required so that we can actually update the `Distribution` object after each optimization step.* - -*Alternatively, we can instead provide the mapping $$\theta \mapsto q_{\theta}$$ directly together with initial parameters using the signature `vi(m, advi, getq, θ_init)` as mentioned earlier. We'll see an explicit example of this later on!* - -To compute statistics for our approximation we need samples: - - -```julia -z = rand(q, 10_000); -``` - -Now we can for example look at the average - - -```julia -avg = vec(mean(z; dims = 2)) -``` - - - - - 12-element Array{Float64,1}: - 0.4606389176400052 - 0.05202909837745655 - 0.4064267006145497 - -0.11468688188714653 - -0.09745310785481277 - 0.6148587707658169 - 0.01308179579131569 - 0.09698898180610954 - -0.07232304322690832 - 0.13320265040493984 - 0.28561578772443025 - -0.829825963610117 - - - -The vector has the same ordering as the model, e.g. in this case `σ₂` has index `1`, `intercept` has index `2` and `coefficients` has indices `3:12`. If you forget or you might want to do something programmatically with the result, you can obtain the `sym → indices` mapping as follows: - - -```julia -_, sym2range = Variational.bijector(m; sym_to_ranges = Val(true)); -sym2range -``` - - - - - (intercept = UnitRange{Int64}[2:2], σ₂ = UnitRange{Int64}[1:1], coefficients = UnitRange{Int64}[3:12]) - - - - -```julia -avg[union(sym2range[:σ₂]...)] -``` - - - - - 1-element Array{Float64,1}: - 0.4606389176400052 - - - - -```julia -avg[union(sym2range[:intercept]...)] -``` - - - - - 1-element Array{Float64,1}: - 0.05202909837745655 - - - - -```julia -avg[union(sym2range[:coefficients]...)] -``` - - - - - 10-element Array{Float64,1}: - 0.4064267006145497 - -0.11468688188714653 - -0.09745310785481277 - 0.6148587707658169 - 0.01308179579131569 - 0.09698898180610954 - -0.07232304322690832 - 0.13320265040493984 - 0.28561578772443025 - -0.829825963610117 - - - -*Note: as you can see, this is slightly awkward to work with at the moment. We'll soon add a better way of dealing with this.* - -With a bit of work (this will be much easier in the future), we can also visualize the approximate marginals of the different variables, similar to `plot(chain)`: - - -```julia -function plot_variational_marginals(z, sym2range) - ps = [] - - for (i, sym) in enumerate(keys(sym2range)) - indices = union(sym2range[sym]...) # <= array of ranges - if sum(length.(indices)) > 1 - offset = 1 - for r in indices - for j in r - p = density(z[j, :], title = "$$(sym)[$$offset]", titlefontsize = 10, label = "") - push!(ps, p) - - offset += 1 - end - end - else - p = density(z[first(indices), :], title = "$$(sym)", titlefontsize = 10, label = "") - push!(ps, p) - end - end - - return plot(ps..., layout = (length(ps), 1), size = (500, 1500)) -end -``` - - - - - plot_variational_marginals (generic function with 1 method) - - - - -```julia -plot_variational_marginals(z, sym2range) -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_68_0.png) - - - -And let's compare this to using the `NUTS` sampler: - - -```julia -chain = sample(m, NUTS(0.65), 10_000); -``` - - ┌ Info: Found initial step size - │ ϵ = 0.4 - └ @ Turing.Inference /home/cameron/.julia/packages/Turing/cReBm/src/inference/hmc.jl:556 - Sampling: 100%|█████████████████████████████████████████| Time: 0:00:04 - - - -```julia -plot(chain) -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_71_0.png) - - - - -```julia -vi_mean = vec(mean(z; dims = 2))[[union(sym2range[:coefficients]...)..., union(sym2range[:intercept]...)..., union(sym2range[:σ₂]...)...]] -``` - - - - - 12-element Array{Float64,1}: - 0.4064267006145497 - -0.11468688188714653 - -0.09745310785481277 - 0.6148587707658169 - 0.01308179579131569 - 0.09698898180610954 - -0.07232304322690832 - 0.13320265040493984 - 0.28561578772443025 - -0.829825963610117 - 0.05202909837745655 - 0.4606389176400052 - - - - -```julia -mean(chain).nt.mean -``` - - - - - 12-element Array{Float64,1}: - 0.40737234076000634 - -0.12119407949255825 - -0.09258229213058687 - 0.6075161662165318 - 0.010710254061742489 - 0.0962666098260057 - -0.07340041375352217 - 0.14124748712473906 - 0.2782293300542158 - -0.8234179979734787 - 0.049650076749642606 - 0.47011974512236054 - - - -One thing we can look at is simply the squared error between the means: - - -```julia -sum(abs2, mean(chain).nt.mean .- vi_mean) -``` - - - - - 0.00038407031406971286 - - - -That looks pretty good! But let's see how the predictive distributions looks for the two. - -## Prediction - -Similarily to the linear regression tutorial, we're going to compare to multivariate ordinary linear regression using the `GLM` package: - - -```julia -# Import the GLM package. -using GLM - -# Perform multivariate OLS. -ols = lm(@formula(MPG ~ Cyl + Disp + HP + DRat + WT + QSec + VS + AM + Gear + Carb), train_cut) - -# Store our predictions in the original dataframe. -train_cut.OLSPrediction = unstandardize(GLM.predict(ols), data.MPG); -test_cut.OLSPrediction = unstandardize(GLM.predict(ols, test_cut), data.MPG); -``` - - -```julia -# Make a prediction given an input vector. -function prediction_chain(chain, x) - p = get_params(chain) - α = mean(p.intercept) - β = collect(mean.(p.coefficients)) - return α .+ x * β -end -``` - - - - - prediction_chain (generic function with 1 method) - - - - -```julia -# Make a prediction using samples from the variational posterior given an input vector. -function prediction(samples::AbstractVector, sym2ranges, x) - α = mean(samples[union(sym2ranges[:intercept]...)]) - β = vec(mean(samples[union(sym2ranges[:coefficients]...)]; dims = 2)) - return α .+ x * β -end - -function prediction(samples::AbstractMatrix, sym2ranges, x) - α = mean(samples[union(sym2ranges[:intercept]...), :]) - β = vec(mean(samples[union(sym2ranges[:coefficients]...), :]; dims = 2)) - return α .+ x * β -end -``` - - - - - prediction (generic function with 2 methods) - - - - -```julia -# Unstandardize the dependent variable. -train_cut.MPG = unstandardize(train_cut.MPG, data.MPG); -test_cut.MPG = unstandardize(test_cut.MPG, data.MPG); -``` - - -```julia -# Show the first side rows of the modified dataframe. -first(test_cut, 6) -``` - - - - -

6 rows × 12 columns (omitted printing of 4 columns)

MPGCylDispHPDRatWTQSecVS
Float64Float64Float64Float64Float64Float64Float64Float64
1116.1951.014880.5912450.0483133-0.8351980.222544-0.307089-0.868028
2114.2951.014880.9623961.43390.2495660.636461-1.36476-0.868028
3120.1951.014881.365820.412942-0.9661180.641571-0.446992-0.868028
4128.295-1.22486-1.22417-1.176840.904164-1.310480.5882951.11604
5126.995-1.22486-0.890939-0.8122111.55876-1.10097-0.642858-0.868028
6131.395-1.22486-1.09427-0.4913370.324377-1.74177-0.5309351.11604
- - - - -```julia -z = rand(q, 10_000); -``` - - -```julia -# Calculate the predictions for the training and testing sets using the samples `z` from variational posterior -train_cut.VIPredictions = unstandardize(prediction(z, sym2range, train), data.MPG); -test_cut.VIPredictions = unstandardize(prediction(z, sym2range, test), data.MPG); - -train_cut.BayesPredictions = unstandardize(prediction_chain(chain, train), data.MPG); -test_cut.BayesPredictions = unstandardize(prediction_chain(chain, test), data.MPG); -``` - - -```julia -vi_loss1 = mean((train_cut.VIPredictions - train_cut.MPG).^2) -bayes_loss1 = mean((train_cut.BayesPredictions - train_cut.MPG).^2) -ols_loss1 = mean((train_cut.OLSPrediction - train_cut.MPG).^2) - -vi_loss2 = mean((test_cut.VIPredictions - test_cut.MPG).^2) -bayes_loss2 = mean((test_cut.BayesPredictions - test_cut.MPG).^2) -ols_loss2 = mean((test_cut.OLSPrediction - test_cut.MPG).^2) - -println("Training set: - VI loss: $$vi_loss1 - Bayes loss: $$bayes_loss1 - OLS loss: $$ols_loss1 -Test set: - VI loss: $$vi_loss2 - Bayes loss: $$bayes_loss2 - OLS loss: $$ols_loss2") -``` - - Training set: - VI loss: 3.0784608943296643 - Bayes loss: 3.0716118391411906 - OLS loss: 3.070926124893019 - Test set: - VI loss: 27.159605003619333 - Bayes loss: 26.58835451660728 - OLS loss: 27.094813070760107 - - -Interestingly the squared difference between true- and mean-prediction on the test-set is actually *better* for the mean-field variational posterior than for the "true" posterior obtained by MCMC sampling using `NUTS`. But, as Bayesians, we know that the mean doesn't tell the entire story. One quick check is to look at the mean predictions ± standard deviation of the two different approaches: - - -```julia -z = rand(q, 1000); -preds = hcat([unstandardize(prediction(z[:, i], sym2range, test), data.MPG) for i = 1:size(z, 2)]...); - -scatter(1:size(test, 1), mean(preds; dims = 2), yerr=std(preds; dims = 2), label="prediction (mean ± std)", size = (900, 500)) -scatter!(1:size(test, 1), unstandardize(test_label, data.MPG), label="true") -xaxis!(1:size(test, 1)) -ylims!(95, 140) -title!("Mean-field ADVI (Normal)") -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_88_0.png) - - - - -```julia -preds = hcat([unstandardize(prediction_chain(chain[i], test), data.MPG) for i = 1:5:size(chain, 1)]...); - -scatter(1:size(test, 1), mean(preds; dims = 2), yerr=std(preds; dims = 2), label="prediction (mean ± std)", size = (900, 500)) -scatter!(1:size(test, 1), unstandardize(test_label, data.MPG), label="true") -xaxis!(1:size(test, 1)) -ylims!(95, 140) -title!("MCMC (NUTS)") -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_89_0.png) - - - -Indeed we see that the MCMC approach generally provides better uncertainty estimates than the mean-field ADVI approach! Good. So all the work we've done to make MCMC fast isn't for nothing. - -## Alternative: provide parameter-to-distribution instead of `q` with`update` implemented - -As mentioned earlier, it's also possible to just provide the mapping $$\theta \mapsto q_{\theta}$$ rather than the variational family / initial variational posterior `q`, i.e. use the interface `vi(m, advi, getq, θ_init)` where `getq` is the mapping $$\theta \mapsto q_{\theta}$$ - -In this section we're going to construct a mean-field approximation to the model by hand using a composition of`Shift` and `Scale` from Bijectors.jl togheter with a standard multivariate Gaussian as the base distribution. - - -```julia -using Bijectors -``` - - -```julia -using Bijectors: Scale, Shift -``` - - -```julia -d = length(q) -base_dist = Turing.DistributionsAD.TuringDiagMvNormal(zeros(d), ones(d)) -``` - - - - - DistributionsAD.TuringDiagMvNormal{Array{Float64,1},Array{Float64,1}}( - m: [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0] - σ: [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0] - ) - - - - -`bijector(model::Turing.Model)` is defined by Turing, and will return a `bijector` which takes you from the space of the latent variables to the real space. In this particular case, this is a mapping `((0, ∞) × ℝ × ℝ¹⁰) → ℝ¹²`. We're interested in using a normal distribution as a base-distribution and transform samples to the latent space, thus we need the inverse mapping from the reals to the latent space: - - -```julia -to_constrained = inv(bijector(m)); -``` - - -```julia -function getq(θ) - d = length(θ) ÷ 2 - A = @inbounds θ[1:d] - b = @inbounds θ[d + 1: 2 * d] - - b = to_constrained ∘ Shift(b; dim = Val(1)) ∘ Scale(exp.(A); dim = Val(1)) - - return transformed(base_dist, b) -end -``` - - - - - getq (generic function with 1 method) - - - - -```julia -q_mf_normal = vi(m, advi, getq, randn(2 * d)); -``` - - ┌ Info: [ADVI] Should only be seen once: optimizer created for θ - │ objectid(θ) = 8127634262038331167 - └ @ Turing.Variational /home/cameron/.julia/packages/Turing/cReBm/src/variational/VariationalInference.jl:204 - [ADVI] Optimizing...: 100%|█████████████████████████████████████████| Time: 0:00:06 - - - -```julia -p1 = plot_variational_marginals(rand(q_mf_normal, 10_000), sym2range) # MvDiagNormal + Affine transformation + to_constrained -p2 = plot_variational_marginals(rand(q, 10_000), sym2range) # Turing.meanfield(m) - -plot(p1, p2, layout = (1, 2), size = (800, 2000)) -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_100_0.png) - - - -As expected, the fits look pretty much identical. - -But using this interface it becomes trivial to go beyond the mean-field assumption we made for the variational posterior, as we'll see in the next section. - -### Relaxing the mean-field assumption - -Here we'll instead consider the variational family to be a full non-diagonal multivariate Gaussian. As in the previous section we'll implement this by transforming a standard multivariate Gaussian using `Scale` and `Shift`, but now `Scale` will instead be using a lower-triangular matrix (representing the Cholesky of the covariance matrix of a multivariate normal) in constrast to the diagonal matrix we used in for the mean-field approximate posterior. - - -```julia -using LinearAlgebra -``` - - -```julia -d = 12 - -function getq(θ) - offset = 0 - L = LowerTriangular(reshape(@inbounds(θ[offset + 1: offset + d^2]), (d, d))) - offset += d^2 - b = @inbounds θ[offset + 1: offset + d] - - # For this to represent a covariance matrix we need to ensure that the diagonal is positive. - # We can enforce this by zeroing out the diagonal and then adding back the diagonal exponentiated. - D = Diagonal(diag(L)) - A = L - D + exp(D) # exp for Diagonal is the same as exponentiating only the diagonal entries - - b = to_constrained ∘ Shift(b; dim = Val(1)) ∘ Scale(A; dim = Val(1)) - - return transformed(base_dist, b) -end -``` - - - - - getq (generic function with 1 method) - - - - -```julia -advi = ADVI(10, 20_000) -``` - - - - - ADVI{Turing.Core.ForwardDiffAD{40}}(10, 20000) - - - - -```julia -q_full_normal = vi(m, advi, getq, randn(d^2 + d); optimizer = Variational.DecayedADAGrad(1e-2)); -``` - - [ADVI] Optimizing...: 100%|█████████████████████████████████████████| Time: 0:01:05 - - -Let's have a look at the learned covariance matrix: - - -```julia -A = q_full_normal.transform.ts[1].a -``` - - - - - 12×12 LowerTriangular{Float64,Array{Float64,2}}: - 0.154572 ⋅ ⋅ … ⋅ ⋅ ⋅ - 0.00674249 0.169072 ⋅ ⋅ ⋅ ⋅ - -0.00288782 -0.0283984 0.413288 ⋅ ⋅ ⋅ - -0.030621 0.0450533 -0.0415525 ⋅ ⋅ ⋅ - -0.0115003 0.208366 -0.0420414 ⋅ ⋅ ⋅ - 0.00139553 -0.0619506 0.0853589 … ⋅ ⋅ ⋅ - 0.0129097 -0.0647154 0.00228644 ⋅ ⋅ ⋅ - -0.0128701 -0.0531755 0.0999936 ⋅ ⋅ ⋅ - 0.00169318 0.0274239 0.0903744 ⋅ ⋅ ⋅ - -0.0172387 -0.0304655 0.0661713 0.14843 ⋅ ⋅ - -0.000468924 0.300281 0.0789093 … -0.131391 0.128256 ⋅ - 0.00160201 -0.122274 -0.0776935 0.0468996 -0.00752499 0.120458 - - - - -```julia -heatmap(cov(A * A')) -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_110_0.png) - - - - -```julia -zs = rand(q_full_normal, 10_000); -``` - - -```julia -p1 = plot_variational_marginals(rand(q_mf_normal, 10_000), sym2range) -p2 = plot_variational_marginals(rand(q_full_normal, 10_000), sym2range) - -plot(p1, p2, layout = (1, 2), size = (800, 2000)) -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_112_0.png) - - - -So it seems like the "full" ADVI approach, i.e. no mean-field assumption, obtain the same modes as the mean-field approach but with greater uncertainty for some of the `coefficients`. This - - -```julia -# Unfortunately, it seems like this has quite a high variance which is likely to be due to numerical instability, -# so we consider a larger number of samples. If we get a couple of outliers due to numerical issues, -# these kind affect the mean prediction greatly. -z = rand(q_full_normal, 10_000); -``` - - -```julia -train_cut.VIFullPredictions = unstandardize(prediction(z, sym2range, train), data.MPG); -test_cut.VIFullPredictions = unstandardize(prediction(z, sym2range, test), data.MPG); -``` - - -```julia -vi_loss1 = mean((train_cut.VIPredictions - train_cut.MPG).^2) -vifull_loss1 = mean((train_cut.VIFullPredictions - train_cut.MPG).^2) -bayes_loss1 = mean((train_cut.BayesPredictions - train_cut.MPG).^2) -ols_loss1 = mean((train_cut.OLSPrediction - train_cut.MPG).^2) - -vi_loss2 = mean((test_cut.VIPredictions - test_cut.MPG).^2) -vifull_loss2 = mean((test_cut.VIFullPredictions - test_cut.MPG).^2) -bayes_loss2 = mean((test_cut.BayesPredictions - test_cut.MPG).^2) -ols_loss2 = mean((test_cut.OLSPrediction - test_cut.MPG).^2) - -println("Training set: - VI loss: $$vi_loss1 - VI (full) loss: $$vifull_loss1 - Bayes loss: $$bayes_loss1 - OLS loss: $$ols_loss1 -Test set: - VI loss: $$vi_loss2 - VI (full) loss: $$vifull_loss2 - Bayes loss: $$bayes_loss2 - OLS loss: $$ols_loss2") -``` - - Training set: - VI loss: 3.0784608943296643 - VI (full) loss: 3.0926834377972288 - Bayes loss: 3.0716118391411906 - OLS loss: 3.070926124893019 - Test set: - VI loss: 27.159605003619333 - VI (full) loss: 26.912162732716684 - Bayes loss: 26.58835451660728 - OLS loss: 27.094813070760107 - - - -```julia -z = rand(q_mf_normal, 1000); -preds = hcat([unstandardize(prediction(z[:, i], sym2range, test), data.MPG) for i = 1:size(z, 2)]...); - -p1 = scatter(1:size(test, 1), mean(preds; dims = 2), yerr=std(preds; dims = 2), label="prediction (mean ± std)", size = (900, 500)) -scatter!(1:size(test, 1), unstandardize(test_label, data.MPG), label="true") -xaxis!(1:size(test, 1)) -ylims!(95, 140) -title!("Mean-field ADVI (Normal)") -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_117_0.png) - - - - -```julia -z = rand(q_full_normal, 1000); -preds = hcat([unstandardize(prediction(z[:, i], sym2range, test), data.MPG) for i = 1:size(z, 2)]...); - -p2 = scatter(1:size(test, 1), mean(preds; dims = 2), yerr=std(preds; dims = 2), label="prediction (mean ± std)", size = (900, 500)) -scatter!(1:size(test, 1), unstandardize(test_label, data.MPG), label="true") -xaxis!(1:size(test, 1)) -ylims!(95, 140) -title!("Full ADVI (Normal)") -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_118_0.png) - - - - -```julia -preds = hcat([unstandardize(prediction_chain(chain[i], test), data.MPG) for i = 1:5:size(chain, 1)]...); - -p3 = scatter(1:size(test, 1), mean(preds; dims = 2), yerr=std(preds; dims = 2), label="prediction (mean ± std)", size = (900, 500)) -scatter!(1:size(test, 1), unstandardize(test_label, data.MPG), label="true") -xaxis!(1:size(test, 1)) -ylims!(95, 140) -title!("MCMC (NUTS)") -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_119_0.png) - - - - -```julia -plot(p1, p2, p3, layout = (1, 3), size = (900, 250), label="") -``` - - - - -![png](../9_VariationalInference_files/9_VariationalInference_120_0.png) - - - -Here we actually see that indeed both the full ADVI and the MCMC approaches does a much better job of quantifying the uncertainty of predictions for never-before-seen samples, with full ADVI seemingly *overestimating* the variance slightly compared to MCMC. - -So now you know how to do perform VI on your Turing.jl model! Great isn't it? diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_100_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_100_0.png deleted file mode 100644 index ec54a783e..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_100_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_110_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_110_0.png deleted file mode 100644 index 3fcd65315..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_110_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_112_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_112_0.png deleted file mode 100644 index f959dd373..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_112_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_117_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_117_0.png deleted file mode 100644 index 9de8e1aed..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_117_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_118_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_118_0.png deleted file mode 100644 index fa507ddb9..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_118_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_119_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_119_0.png deleted file mode 100644 index fe7e866ef..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_119_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_120_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_120_0.png deleted file mode 100644 index 239e7adc8..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_120_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_34_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_34_0.png deleted file mode 100644 index 35fc8f609..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_34_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_36_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_36_0.png deleted file mode 100644 index a137bf68d..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_36_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_36_1.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_36_1.png deleted file mode 100644 index 31c903157..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_36_1.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_68_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_68_0.png deleted file mode 100644 index 7b28b445d..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_68_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_71_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_71_0.png deleted file mode 100644 index 4b99f36bf..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_71_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_88_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_88_0.png deleted file mode 100644 index b0635a11d..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_88_0.png and /dev/null differ diff --git a/_tutorials/9_VariationalInference_files/9_VariationalInference_89_0.png b/_tutorials/9_VariationalInference_files/9_VariationalInference_89_0.png deleted file mode 100644 index fe7e866ef..000000000 Binary files a/_tutorials/9_VariationalInference_files/9_VariationalInference_89_0.png and /dev/null differ diff --git a/_tutorials/figures/3_BayesNN_12_1.png b/_tutorials/figures/3_BayesNN_12_1.png deleted file mode 100644 index 6b1574a08..000000000 Binary files a/_tutorials/figures/3_BayesNN_12_1.png and /dev/null differ diff --git a/_tutorials/figures/3_BayesNN_12_1.svg b/_tutorials/figures/3_BayesNN_12_1.svg deleted file mode 100644 index b2727920d..000000000 --- a/_tutorials/figures/3_BayesNN_12_1.svg +++ /dev/null @@ -1,369 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - - - - - -y1 - - - - -y2 - - \ No newline at end of file diff --git a/_tutorials/figures/3_BayesNN_2_1.png b/_tutorials/figures/3_BayesNN_2_1.png deleted file mode 100644 index 6dfe91640..000000000 Binary files a/_tutorials/figures/3_BayesNN_2_1.png and /dev/null differ diff --git a/_tutorials/figures/3_BayesNN_2_1.svg b/_tutorials/figures/3_BayesNN_2_1.svg deleted file mode 100644 index 28f921904..000000000 --- a/_tutorials/figures/3_BayesNN_2_1.svg +++ /dev/null @@ -1,249 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --4 - - --2 - - -0 - - -2 - - -4 - - --4 - - --2 - - -0 - - -2 - - -4 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -y1 - - - - -y2 - - \ No newline at end of file diff --git a/_tutorials/figures/3_BayesNN_7_1.png b/_tutorials/figures/3_BayesNN_7_1.png deleted file mode 100644 index da5b768c3..000000000 Binary files a/_tutorials/figures/3_BayesNN_7_1.png and /dev/null differ diff --git a/_tutorials/figures/3_BayesNN_7_1.svg b/_tutorials/figures/3_BayesNN_7_1.svg deleted file mode 100644 index ec0757c53..000000000 --- a/_tutorials/figures/3_BayesNN_7_1.svg +++ /dev/null @@ -1,357 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - - - - - -y1 - - - - -y2 - - \ No newline at end of file diff --git a/_tutorials/figures/3_BayesNN_9_1.png b/_tutorials/figures/3_BayesNN_9_1.png deleted file mode 100644 index 0662a2e99..000000000 Binary files a/_tutorials/figures/3_BayesNN_9_1.png and /dev/null differ diff --git a/_tutorials/figures/3_BayesNN_9_1.svg b/_tutorials/figures/3_BayesNN_9_1.svg deleted file mode 100644 index f310183f0..000000000 --- a/_tutorials/figures/3_BayesNN_9_1.svg +++ /dev/null @@ -1,356 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - --5.0 - - --2.5 - - -0.0 - - -2.5 - - -5.0 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - -0 - - -0.1 - - -0.2 - - -0.3 - - -0.4 - - -0.5 - - -0.6 - - -0.7 - - -0.8 - - -0.9 - - -1.0 - - - - - - - - - - -y1 - - - - -y2 - - \ No newline at end of file diff --git a/make-utils.jl b/make-utils.jl index fc35364fa..1bbd3a339 100644 --- a/make-utils.jl +++ b/make-utils.jl @@ -250,8 +250,6 @@ function copy_tutorial(tutorial_path) fix_image_path(target_path) end end - index = joinpath(@__DIR__, "src/tutorials/index.md") - cp(index, tutorial_path * "/index.md", force=true) catch e rethrow(e) finally @@ -259,8 +257,8 @@ function copy_tutorial(tutorial_path) end end -function with_baseurl(func, baseurl) - jekyll_config = joinpath(@__DIR__, "_config.yml") +function with_baseurl(func, baseurl, config_path) + jekyll_config = config_path lines = readlines(jekyll_config, keep=true) open(jekyll_config, "w+") do f for line in lines diff --git a/make.jl b/make.jl index dadd4dc25..b510e2238 100644 --- a/make.jl +++ b/make.jl @@ -1,15 +1,64 @@ +import Pkg +Pkg.activate(".") + using Documenter, DocumenterMarkdown, Turing, AdvancedHMC, Bijectors, AdvancedMH using LibGit2: clone, tag_list, GitRepo +using Documenter: GitHubActions # Include the utility functions. include("make-utils.jl") -# Find the package directory +# Get the version number +trim_version(x) = x[1:findlast('.', x) - 1] + +version, is_dev = if haskey(ENV, "TURING_VERSION") + ENV["TURING_VERSION"], true +else + if length(ARGS) > 0 + trim_version(ARGS[1]), false + else + "dev", true + end +end + +# Make a temporary folder to build from +tmp_path = mktempdir() + +# Paths +## The location of the package to build (Turing()) package_directory = dirname(dirname(pathof(Turing))) +## The location of the docs folder inside that package +docs_path = joinpath(package_directory, "docs") +## The src files for the docs -- markdown documents, typically. source_path = joinpath(package_directory, "docs", "src") +## The path of turing.ml that we are running this code from. +local_path = @__DIR__ +## The place to put the files from source_path after they go through Documenter.jl +build_path = joinpath(tmp_path, "_docs") + +# Get any files from Turing's directory +for (root, dirs, files) in walkdir(docs_path) + new_root = replace(root, docs_path => tmp_path) + + for file in files + old_file = joinpath(root, file) + new_file = joinpath(new_root, file) + @debug "" old_file new_file + if !isdir(dirname(new_file)) + mkpath(dirname(new_file)) + end + cp(old_file, new_file) + end +end -# Paths. -build_path = joinpath(@__DIR__, "_docs") +# Copy all the local files to the temporary path +paths = readdir(local_path, join=true) +filter!(x -> !(basename(x) in ["make.jl", "make-utils.jl"]), paths) +for path in paths + new_path = replace(path, local_path => tmp_path) + @debug "" path new_path + cp(path, new_path, force=true) +end # Build docs with_clean_docs(source_path, build_path) do source, build @@ -24,53 +73,35 @@ end # You can skip this part if you are on a metered # connection by calling `julia make.jl no-tutorials` -tutorial_path = joinpath(@__DIR__, "_tutorials") +tutorial_path = joinpath(tmp_path, "_tutorials") in("no-tutorials", ARGS) || copy_tutorial(tutorial_path) # set default baseurl for the master branch -baseurl = "/dev" - -# deploy -devurl = "dev" -tags = tag_list(GitRepo(package_directory)) -versions = VersionNumber.(tags) - -# Find the most recent tag -function highest_tags(versions) - majors = unique(map(x -> x.major, versions)) - tags_to_use = [] - - for major in majors - filtered = filter(x -> x.major == major, versions) - minors = unique(map(x -> x.minor, filtered)) - - for minor in minors - minor_filtered = sort(filter(x -> x.minor == minor, filtered)) - - push!(tags_to_use, minor_filtered[end]) - end - end - return sort(tags_to_use) -end +baseurl = "/" * version -highest = maximum(highest_tags(versions)) +@info "" baseurl -# set baseurl for version tag when current head is tagged -vtag = "v" * string(highest) +# deploy +old_jekyll_build = joinpath(local_path, "jekyll-build") +new_jekyll_build = joinpath(tmp_path, "jekyll-build") -version_match = match(r"^(v\d+\.\d+\.\d+)$", vtag) -if !isnothing(version_match) - baseurl = "/" * version_match[1] -end +# Move jekyll-build to the temporary path +cp(old_jekyll_build, new_jekyll_build, force=true) +with_baseurl(() -> run(`$new_jekyll_build`), baseurl, joinpath(local_path, "_config.yml")) +repo = "github.com:TuringLang/turing.ml.git" -jekyll_build = joinpath(@__DIR__, "jekyll-build") -with_baseurl(() -> run(`$jekyll_build`), baseurl) +deploy_config = GitHubActions( + "TuringLang/turing.ml", #github_repository::String + "push", #github_event_name::String + is_dev ? "refs/branch/master" : "refs/tags/$(ARGS[1])" #github_ref::String +) deploydocs( target = "_site", repo = repo, branch = "gh-pages", devbranch = "master", - devurl = devurl, - versions = ["stable" => "v^", "v#.#", devurl => devurl] + devurl = "dev", + versions = ["stable" => "v^", "v#.#", "dev" => "dev"], + deploy_config = deploy_config ) diff --git a/src/contributing/guide.md b/src/contributing/guide.md deleted file mode 100644 index a0d23f9df..000000000 --- a/src/contributing/guide.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Contributing ---- - -# Contributing - -Turing is an open source project. If you feel that you have some relevant skills and are interested in contributing, then please do get in touch. You can contribute by opening issues on GitHub or implementing things yourself and making a pull request. We would also appreciate example models written using Turing. - - -Turing has a [style guide]({{site.baseurl}}/docs/contributing/style-guide). It is not strictly necessary to review it before making a pull request, but you may be asked to change portions of your code to conform with the style guide before it is merged. - - -## How to Contribute - - -### Getting Started - - - * [Fork this repository](https://github.com/TuringLang/Turing.jl#fork-destination-box). - * Clone your fork on your local machine: `git clone https://github.com/your_username/Turing.jl`. - * Add a remote corresponding to this repository: - - -`git remote add upstream https://github.com/TuringLang/Turing.jl`. - - -### What Can I Do? - - -Look at the [issues](https://github.com/TuringLang/Turing.jl/issues) page to find an outstanding issue. For instance, you could implement new features, fix bugs or write example models. - - -### Git Workflow - - -For more information on how the Git workflow typically functions, please see the [GitHub's introduction](https://guides.github.com/introduction/flow/) or [Julia's contribution guide](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md). - diff --git a/src/contributing/style-guide.md b/src/contributing/style-guide.md deleted file mode 100644 index 15991d6bc..000000000 --- a/src/contributing/style-guide.md +++ /dev/null @@ -1,662 +0,0 @@ ---- -title: Style Guide ---- - -# Style Guide - -This style guide is adapted from [Invenia](https://invenia.ca/labs/)'s style guide. We would like to thank them for allowing us to access and use it. Please don't let not having read it stop you from contributing to Turing! No one will be annoyed if you open a PR whose style doesn't follow these conventions; we will just help you correct it before it gets merged. - - -These conventions were originally written at Invenia, taking inspiration from a variety of sources including Python's [PEP8](http://legacy.python.org/dev/peps/pep-0008), Julia's [Notes for Contributors](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md), and Julia's [Style Guide](https://docs.julialang.org/en/latest/manual/style-guide/). - - -What follows is a mixture of a verbatim copy of Invenia's original guide and some of our own modifications. - - -## A Word on Consistency - - -When adhering to this style it's important to realize that these are guidelines and not rules. This is [stated best in the PEP8](http://legacy.python.org/dev/peps/pep-0008/#a-foolish-consistency-is-the-hobgoblin-of-little-minds): - - -> A style guide is about consistency. Consistency with this style guide is important. Consistency within a project is more important. Consistency within one module or function is most important. - - - -> But most importantly: know when to be inconsistent – sometimes the style guide just doesn't apply. When in doubt, use your best judgment. Look at other examples and decide what looks best. And don't hesitate to ask! - - - -## Synopsis - - -Attempt to follow both the [Julia Contribution Guidelines](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md#general-formatting-guidelines-for-julia-code-contributions), the [Julia Style Guide](https://docs.julialang.org/en/latest/manual/style-guide/), and this guide. When convention guidelines conflict this guide takes precedence (known conflicts will be noted in this guide). - - - * Use 4 spaces per indentation level, no tabs. - * Try to adhere to a 92 character line length limit. - * Use upper camel case convention for [modules](https://docs.julialang.org/en/latest/manual/modules/) and [types](https://docs.julialang.org/en/latest/manual/types/). - * Use lower case with underscores for method names (note: Julia code likes to use lower case without underscores). - * Comments are good, try to explain the intentions of the code. - * Use whitespace to make the code more readable. - * No whitespace at the end of a line (trailing whitespace). - * Avoid padding brackets with spaces. ex. `Int64(value)` preferred over `Int64( value )`. - - -## Editor Configuration - - -### Sublime Text Settings - - -If you are a user of Sublime Text we recommend that you have the following options in your Julia syntax specific settings. To modify these settings first open any Julia file (`*.jl`) in Sublime Text. Then navigate to: `Preferences > Settings - More > Syntax Specific - User` - - -```json -{ - "translate_tabs_to_spaces": true, - "tab_size": 4, - "trim_trailing_white_space_on_save": true, - "ensure_newline_at_eof_on_save": true, - "rulers": [92] -} -``` - - -### Vim Settings - - -If you are a user of Vim we recommend that you add the following options to your `.vimrc` file. - - -``` -set tabstop=4 " Sets tabstops to a width of four columns. -set softtabstop=4 " Determines the behaviour of TAB and BACKSPACE keys with expandtab. -set shiftwidth=4 " Determines the results of >>, <<, and ==. - -au FileType julia setlocal expandtab " Replaces tabs with spaces. -au FileType julia setlocal colorcolumn=93 " Highlights column 93 to help maintain the 92 character line limit. -``` - - -By default, Vim seems to guess that `.jl` files are written in Lisp. To ensure that Vim recognizes Julia files you can manually have it check for the `.jl` extension, but a better solution is to install [Julia-Vim](https://github.com/JuliaLang/julia-vim), which also includes proper syntax highlighting and a few cool other features. - - -### Atom Settings - - -Atom defaults preferred line length to 80 characters. We want that at 92 for julia. To change it: - - -1. Go to `Atom -> Preferences -> Packages`. -2. Search for the "language-julia" package and open the settings for it. -3. Find preferred line length (under "Julia Grammar") and change it to 92. - - -## Code Formatting - - -### Function Naming - - -Names of functions should describe an action or property irrespective of the type of the argument; the argument's type provides this information instead. For example, `buyfood(food)` should be `buy(food::Food)`. - - -Names of functions should usually be limited to one or two lowercase words. Ideally write `buyfood` not `buy_food`, but if you are writing a function whose name is hard to read without underscores then please do use them. - - -### Method Definitions - - -Only use short-form function definitions when they fit on a single line: - - -```julia -# Yes: -foo(x::Int64) = abs(x) + 3 -# No: -foobar(array_data::AbstractArray{T}, item::T) where {T<:Int64} = T[ - abs(x) * abs(item) + 3 for x in array_data -] - -# No: -foobar( - array_data::AbstractArray{T}, - item::T, -) where {T<:Int64} = T[abs(x) * abs(item) + 3 for x in array_data] -# Yes: -function foobar(array_data::AbstractArray{T}, item::T) where T<:Int64 - return T[abs(x) * abs(item) + 3 for x in array_data] -end -``` - - -When using long-form functions [always use the `return` keyword](https://groups.google.com/forum/#!topic/julia-users/4RVR8qQDrUg): - - -```julia -# Yes: -function fnc(x) - result = zero(x) - result += fna(x) - return result -end -# No: -function fnc(x) - result = zero(x) - result += fna(x) -end - -# Yes: -function Foo(x, y) - return new(x, y) -end -# No: -function Foo(x, y) - new(x, y) -end -``` - - -Functions definitions with parameter lines which exceed 92 characters should separate each parameter by a newline and indent by one-level: - - -```julia -# Yes: -function foobar( - df::DataFrame, - id::Symbol, - variable::Symbol, - value::AbstractString, - prefix::AbstractString="", -) - # code -end - -# Ok: -function foobar(df::DataFrame, id::Symbol, variable::Symbol, value::AbstractString, prefix::AbstractString="") - # code -end -# No: -function foobar(df::DataFrame, id::Symbol, variable::Symbol, value::AbstractString, - prefix::AbstractString="") - - # code -end -# No: -function foobar( - df::DataFrame, - id::Symbol, - variable::Symbol, - value::AbstractString, - prefix::AbstractString="", - ) - # code -end -``` - - -### Keyword Arguments - - -When calling a function always separate your keyword arguments from your positional arguments with a semicolon. This avoids mistakes in ambiguous cases (such as splatting a `Dict`). - - -```julia -# Yes: -xy = foo(x; y=3) -# No: -xy = foo(x, y=3) -``` - - -### Whitespace - - -Avoid extraneous whitespace in the following situations: - - - * Immediately inside parentheses, square brackets or braces. - - -```julia -Yes: spam(ham[1], [eggs]) -No: spam( ham[ 1 ], [ eggs ] ) -``` - - - * Immediately before a comma or semicolon: - - -```julia -Yes: if x == 4 @show(x, y); x, y = y, x end -No: if x == 4 @show(x , y) ; x , y = y , x end -``` - - - * When using ranges unless additional operators are used: - - -```julia -Yes: ham[1:9], ham[1:3:9], ham[1:3:end] -No: ham[1: 9], ham[1 : 3: 9] -``` - - -```julia -Yes: ham[lower:upper], ham[lower:step:upper] -Yes: ham[lower + offset : upper + offset] -Yes: ham[(lower + offset):(upper + offset)] -No: ham[lower + offset:upper + offset] -``` - - - * More than one space around an assignment (or other) operator to align it with another: - - -```julia -# Yes: -x = 1 -y = 2 -long_variable = 3 - -# No: -x = 1 -y = 2 -long_variable = 3 -``` - - * When using parametric types: - -```julia -# Yes: -f(a::AbstractArray{T, N}) where {T<:Real, N} = ... -g(a::AbstractArray{<:Real, N}) where {N} = ... - -# No: -f(a::AbstractArray{T, N}) where {T <: Real, N} = ... -g(a::AbstractArray{<: Real, N}) where {N} = ... -``` - - * Always surround these binary operators with a single space on either side: assignment ($=$), [updating operators](https://docs.julialang.org/en/latest/manual/mathematical-operations/#Updating-operators-1) ($+=$, $-=$, etc.), [numeric comparisons operators](https://docs.julialang.org/en/latest/manual/mathematical-operations/#Numeric-Comparisons-1) ($==$, $<$, $>$, $!=$, etc.). Note that this guideline does not apply when performing assignment in method definitions. - - -```julia -Yes: i = i + 1 -No: i=i+1 - -Yes: submitted += 1 -No: submitted +=1 - -Yes: x^2 < y -No: x^2 Int - -Searches the `array` for the `val`. For some reason we don't want to use Julia's -builtin search :) - -# Arguments -- `array::MyArray{T}`: the array to search -- `val::T`: the value to search for - -# Keywords -- `verbose::Bool=true`: print out progress details - -# Returns -- `Int`: the index where `val` is located in the `array` - -# Throws -- `NotFoundError`: I guess we could throw an error if `val` isn't found. -""" -function mysearch(array::AbstractArray{T}, val::T) where T - ... -end -``` - - -If your method contains lots of arguments or keywords you may want to exclude them from the method signature on the first line and instead use `args...` and/or `kwargs...`. - - -```julia -""" - Manager(args...; kwargs...) -> Manager - -A cluster manager which spawns workers. - -# Arguments - -- `min_workers::Integer`: The minimum number of workers to spawn or an exception is thrown -- `max_workers::Integer`: The requested number of worker to spawn - -# Keywords - -- `definition::AbstractString`: Name of the job definition to use. Defaults to the - definition used within the current instance. -- `name::AbstractString`: ... -- `queue::AbstractString`: ... -""" -function Manager(...) - ... -end -``` - - -Feel free to document multiple methods for a function within the same docstring. Be careful to only do this for functions you have defined. - - -```julia -""" - Manager(max_workers; kwargs...) - Manager(min_workers:max_workers; kwargs...) - Manager(min_workers, max_workers; kwargs...) - -A cluster manager which spawns workers. - -# Arguments - -- `min_workers::Int`: The minimum number of workers to spawn or an exception is thrown -- `max_workers::Int`: The number of requested workers to spawn - -# Keywords - -- `definition::AbstractString`: Name of the job definition to use. Defaults to the - definition used within the current instance. -- `name::AbstractString`: ... -- `queue::AbstractString`: ... -""" -function Manager end - -``` - - -If the documentation for bullet-point exceeds 92 characters the line should be wrapped and slightly indented. Avoid aligning the text to the `:`. - - -```julia -""" -... - -# Keywords -- `definition::AbstractString`: Name of the job definition to use. Defaults to the - definition used within the current instance. -""" -``` - - -For additional details on documenting in Julia see the [official documentation](http://docs.julialang.org/en/latest/manual/documentation/). - - -## Test Formatting - - -### Testsets - - -Julia provides [test sets](https://docs.julialang.org/en/latest/stdlib/Test/#Working-with-Test-Sets-1) which allows developers to group tests into logical groupings. Test sets can be nested and ideally packages should only have a single "root" test set. It is recommended that the "runtests.jl" file contains the root test set which contains the remainder of the tests: - - -```julia -@testset "PkgExtreme" begin - include("arithmetic.jl") - include("utils.jl") -end -``` - - -The file structure of the `test` folder should mirror that of the `src` folder. Every file in `src` should have a complementary file in the `test` folder, containing tests relevant to that file's contents. - - -### Comparisons - - -Most tests are written in the form `@test x == y`. Since the `==` function doesn't take types into account tests like the following are valid: `@test 1.0 == 1`. Avoid adding visual noise into test comparisons: - - -```julia -# Yes: -@test value == 0 - -# No: -@test value == 0.0 -``` - - -In cases where you are checking the numerical validity of a model's parameter estimates, please use the `check_numerical` function found in `test/test_utils/numerical_tests.jl`. This function will evaluate a model's parameter estimates using tolerance levels `atol` and `rtol`. Testing will only be performed if you are running the test suite locally or if Travis is executing the "Numerical" testing stage. - - -Here is an example of usage: - - -```julia -# Check that m and s are plus or minus one from 1.5 and 2.2, respectively. -check_numerical(chain, [:m, :s], [1.5, 2.2], atol = 1.0) - -# Checks the estimates for a default gdemo model using values 1.5 and 2.0. -check_gdemo(chain, atol = 0.1) - -# Checks the estimates for a default MoG model. -check_MoGtest_default(chain, atol = 0.1) -``` - diff --git a/src/for-developers/compiler.md b/src/for-developers/compiler.md deleted file mode 100644 index 9cf56452a..000000000 --- a/src/for-developers/compiler.md +++ /dev/null @@ -1,231 +0,0 @@ ---- -title: Turing Compiler Design ---- - -In this section, the current design of Turing's model "compiler" is described which enables Turing to perform various types of Bayesian inference without changing the model definition. The "compiler" is essentially just a macro that rewrites the user's model definition to a function that generates a `Model` struct that Julia's dispatch can operate on and that Julia's compiler can successfully do type inference on for efficient machine code generation. - -# Overview - -The following terminology will be used in this section: - -- `D`: observed data variables conditioned upon in the posterior, -- `P`: parameter variables distributed according to the prior distributions, these will also be referred to as random variables, -- `Model`: a fully defined probabilistic model with input data - -`Turing`'s `@model` macro rewrites the user-provided function definition such that it can be used to instantiate a `Model` by passing in the observed data `D`. - -The following are the main jobs of the `@model` macro: -1. Parse `~` and `.~` lines, e.g. `y .~ Normal.(c*x, 1.0)` -2. Figure out if a variable belongs to the data `D` and or to the parameters `P` -3. Enable the handling of missing data variables in `D` when defining a `Model` and treating them as parameter variables in `P` instead -4. Enable the tracking of random variables using the data structures `VarName` and `VarInfo` -5. Change `~`/`.~` lines with a variable in `P` on the LHS to a call to `tilde_assume` or `dot_tilde_assume` -6. Change `~`/`.~` lines with a variable in `D` on the LHS to a call to `tilde_observe` or `dot_tilde_observe` -7. Enable type stable automatic differentiation of the model using type parameters - - -## The model - -A `model::Model` is a callable struct that one can sample from by calling -```julia -(model::Model)([rng, varinfo, sampler, context]) -``` -where `rng` is a random number generator (default: `Random.GLOBAL_RNG`), `varinfo` is a data structure that stores information -about the random variables (default: `DynamicPPL.VarInfo()`), `sampler` is a sampling algorithm (default: `DynamicPPL.SampleFromPrior()`), -and `context` is a sampling context that can, e.g., modify how the log probability is accumulated (default: `DynamicPPL.DefaultContext()`). - -Sampling resets the log joint probability of `varinfo` and increases the evaluation counter of `sampler`. If `context` is a `LikelihoodContext`, -only the log likelihood will be accumulated. With the `DefaultContext` the log joint probability of `P` and `D` is accumulated. - -The `Model` struct contains the three internal fields `f`, `args` and `defaults`. -When `model::Model` is called, then the internal function `model.f` is called as `model.f(rng, varinfo, sampler, context, model.args...)` -(for multithreaded sampling, instead of `varinfo` a threadsafe wrapper is passed to `model.f`). -The positional and keyword arguments that were passed to the user-defined model function when the model was created are saved as a `NamedTuple` -in `model.args`. The default values of the positional and keyword arguments of the user-defined model functions, if any, are saved as a `NamedTuple` -in `model.defaults`. They are used for constructing model instances with different arguments by the `logprob` and `prob` string macros. - -# Example - -Let's take the following model as an example: - -```julia -@model function gauss(x = missing, y = 1.0, ::Type{TV} = Vector{Float64}) where {TV<:AbstractVector} - if x === missing - x = TV(undef, 3) - end - p = TV(undef, 2) - p[1] ~ InverseGamma(2, 3) - p[2] ~ Normal(0, 1.0) - @. x[1:2] ~ Normal(p[2], sqrt(p[1])) - x[3] ~ Normal() - y ~ Normal(p[2], sqrt(p[1])) -end -``` - -The above call of the `@model` macro defines the function `gauss` with positional arguments `x`, `y`, and `::Type{TV}`, rewritten in -such a way that every call of it returns a `model::Model`. Note that only the function body is modified by the `@model` macro, and the -function signature is left untouched. It is also possible to implement models with keyword arguments such as - -```julia -@model function gauss(::Type{TV} = Vector{Float64}; x = missing, y = 1.0) where {TV<:AbstractVector} - ... -end -``` - -This would allow us to generate a model by calling `gauss(; x = rand(3))`. - -If an argument has a default value `missing`, it is treated as a random variable. For variables which require an intialization because we -need to loop or broadcast over its elements, such as `x` above, the following needs to be done: -```julia -if x === missing - x = ... -end -``` -Note that since `gauss` behaves like a regular function it is possible to define additional dispatches in a second step as well. For -instance, we could achieve the same behaviour by - -```julia -@model function gauss(x, y = 1.0, ::Type{TV} = Vector{Float64}) where {TV<:AbstractVector} - p = TV(undef, 2) - ... -end - -function gauss(::Missing, y = 1.0, ::Type{TV} = Vector{Float64}) where {TV<:AbstractVector} - return gauss(TV(undef, 3), y, TV) -end -``` - -If `x` is sampled as a whole from a distribution and not indexed, e.g., `x ~ Normal(...)` or `x ~ MvNormal(...)`, -there is no need to initialize it in an `if`-block. - -## Step 1: Break up the model definition - -First, the `@model` macro breaks up the user-provided function definition using `DynamicPPL.build_model_info`. This function -returns a dictionary consisting of: -- `allargs_exprs`: The expressions of the positional and keyword arguments, without default values. -- `allargs_syms`: The names of the positional and keyword arguments, e.g., `[:x, :y, :TV]` above. -- `allargs_namedtuple`: An expression that constructs a `NamedTuple` of the positional and keyword arguments, e.g., `:((x = x, y = y, TV = TV))` above. -- `defaults_namedtuple`: An expression that constructs a `NamedTuple` of the default positional and keyword arguments, if any, e.g., `:((x = missing, y = 1, TV = Vector{Float64}))` above. -- `modeldef`: A dictionary with the name, arguments, and function body of the model definition, as returned by `MacroTools.splitdef`. - -## Step 2: Generate the body of the internal model function - -In a second step, `DynamicPPL.generate_mainbody` generates the main part of the transformed function body using the user-provided function body -and the provided function arguments, without default values, for figuring out if a variable denotes an observation or a random variable. -Hereby the function `DynamicPPL.generate_tilde` replaces the `L ~ R` lines in the model and the function `DynamicPPL.generate_dot_tilde` replaces -the `@. L ~ R` and `L .~ R` lines in the model. - -In the above example, `p[1] ~ InverseGamma(2, 3)` is replaced with something similar to -```julia -#= REPL[25]:6 =# -begin - var"##tmpright#323" = InverseGamma(2, 3) - var"##tmpright#323" isa Union{Distribution, AbstractVector{<:Distribution}} || throw(ArgumentError("Right-hand side of a ~ must be subtype of Distribution or a vector of Distributions.")) - var"##vn#325" = (DynamicPPL.VarName)(:p, ((1,),)) - var"##inds#326" = ((1,),) - p[1] = (DynamicPPL.tilde_assume)(_rng, _context, _sampler, var"##tmpright#323", var"##vn#325", var"##inds#326", _varinfo) -end -``` -Here the first line is a so-called line number node that enables more helpful error messages by providing users with the exact location -of the error in their model definition. Then the right hand side (RHS) of the `~` is assigned to a variable (with an automatically generated name). -We check that the RHS is a distribution or an array of distributions, otherwise an error is thrown. -Next we extract a compact representation of the variable with its name and index (or indices). Finally, the `~` expression is replaced with -a call to `DynamicPPL.tilde_assume` since the compiler figured out that `p[1]` is a random variable using the following -heuristic: -1. If the symbol on the LHS of `~`, `:p` in this case, is not among the arguments to the model, `(:x, :y, :T)` in this case, it is a random variable. -2. If the symbol on the LHS of `~`, `:p` in this case, is among the arguments to the model but has a value of `missing`, it is a random variable. -2. If the value of the LHS of `~`, `p[1]` in this case, is `missing`, then it is a random variable. -4. Otherwise, it is treated as an observation. - -The `DynamicPPL.tilde_assume` function takes care of sampling the random variable, if needed, and updating its value and the accumulated log joint -probability in the `_varinfo` object. If `L ~ R` is an observation, `DynamicPPL.tilde_observe` is called with the same arguments except the -random number generator `_rng` (since observations are never sampled). - -A similar transformation is performed for expressions of the form `@. L ~ R` and `L .~ R`. For instance, -`@. x[1:2] ~ Normal(p[2], sqrt(p[1]))` is replaced with -```julia -#= REPL[25]:8 =# -begin - var"##tmpright#331" = Normal.(p[2], sqrt.(p[1])) - var"##tmpright#331" isa Union{Distribution, AbstractVector{<:Distribution}} || throw(ArgumentError("Right-hand side of a ~ must be subtype of Distribution or a vector of Distributions.")) - var"##vn#333" = (DynamicPPL.VarName)(:x, ((1:2,),)) - var"##inds#334" = ((1:2,),) - var"##isassumption#335" = begin - let var"##vn#336" = (DynamicPPL.VarName)(:x, ((1:2,),)) - if !((DynamicPPL.inargnames)(var"##vn#336", _model)) || (DynamicPPL.inmissings)(var"##vn#336", _model) - true - else - x[1:2] === missing - end - end - end - if var"##isassumption#335" - x[1:2] .= (DynamicPPL.dot_tilde_assume)(_rng, _context, _sampler, var"##tmpright#331", x[1:2], var"##vn#333", var"##inds#334", _varinfo) - else - (DynamicPPL.dot_tilde_observe)(_context, _sampler, var"##tmpright#331", x[1:2], var"##vn#333", var"##inds#334", _varinfo) - end -end -``` -The main difference in the expanded code between `L ~ R` and `@. L ~ R` is that the former doesn't assume `L` to be defined, it can be a new Julia variable in the scope, while the latter assumes `L` already exists. Moreover, `DynamicPPL.dot_tilde_assume` and `DynamicPPL.dot_tilde_observe` are called -instead of `DynamicPPL.tilde_assume` and `DynamicPPL.tilde_observe`. - -## Step 3: Replace the user-provided function body - -Finally, we replace the user-provided function body using `DynamicPPL.build_output`. This function uses `MacroTools.combinedef` to reassemble -the user-provided function with a new function body. In the modified function body an anonymous function is created whose function body -was generated in step 2 above and whose arguments are -- a random number generator `_rng`, -- a model `_model`, -- a datastructure `_varinfo`, -- a sampler `_sampler`, -- a sampling context `_context`, -- and all positional and keyword arguments of the user-provided model function as positional arguments -without any default values. Finally, in the new function body a `model::Model` with this anonymous function as internal function is returned. - -# `VarName` - -In order to track random variables in the sampling process, `Turing` uses the struct `VarName{sym}` which acts as a random variable identifier generated at runtime. The `VarName` of a random variable is generated from the expression on the LHS of a `~` statement when the symbol on the LHS is in `P`. Every `vn::VarName{sym}` has a symbol `sym` which is the symbol of the Julia variable in the model that the random variable belongs to. For example, `x[1] ~ Normal()` will generate an instance of `VarName{:x}` assuming `x` is in `P`. Every `vn::VarName` also has a field `indexing` which stores the indices requires to access the random variable from the Julia variable indicated by `sym`. For example, `x[1] ~ Normal()` will generate a `vn::VarName{:x}` with `vn.indexing == "[1]"`. `VarName` also supports hierarchical arrays and range indexing. Some more examples: -- `x[1] ~ Normal()` will generate a `VarName{:x}` with `indexing == "[1]"`. -- `x[:,1] ~ MvNormal(zeros(2))` will generate a `VarName{:x}` with `indexing == "[Colon(),1]"`. -- `x[:,1][2] ~ Normal()` will generate a `VarName{:x}` with `indexing == "[Colon(),1][2]"`. - -# `VarInfo` - -## Overview - -`VarInfo` is the data structure in `Turing` that facilitates tracking random variables and certain metadata about them that are required for sampling. For instance, the distribution of every random variable is stored in `VarInfo` because we need to know the support of every random variable when sampling using HMC for example. Random variables whose distributions have a constrained support are transformed using a bijector from [Bijectors.jl](https://github.com/TuringLang/Bijectors.jl) so that the sampling happens in the unconstrained space. Different samplers require different metadata about the random variables. - -The definition of `VarInfo` in `Turing` is: -``` -struct VarInfo{Tmeta, Tlogp} <: AbstractVarInfo - metadata::Tmeta - logp::Base.RefValue{Tlogp} - num_produce::Base.RefValue{Int} -end -``` -Based on the type of `metadata`, the `VarInfo` is either aliased `UntypedVarInfo` or `TypedVarInfo`. `metadata` can be either a subtype of the union type `Metadata` or a `NamedTuple` of multiple such subtypes. Let `vi` be an instance of `VarInfo`. If `vi isa VarInfo{<:Metadata}`, then it is called an `UntypedVarInfo`. If `vi isa VarInfo{<:NamedTuple}`, then `vi.metadata` would be a `NamedTuple` mapping each symbol in `P` to an instance of `Metadata`. `vi` would then be called a `TypedVarInfo`. The other fields of `VarInfo` include `logp` which is used to accumulate the log probability or log probability density of the variables in `P` and `D`. `num_produce` keeps track of how many observations have been made in the model so far. This is incremented when running a `~` statement when the symbol on the LHS is in `D`. - -## `Metadata` - -The `Metadata` struct stores some metadata about the random variables sampled. This helps -query certain information about a variable such as: its distribution, which samplers -sample this variable, its value and whether this value is transformed to real space or -not. Let `md` be an instance of `Metadata`: -- `md.vns` is the vector of all `VarName` instances. Let `vn` be an arbitrary element of `md.vns` -- `md.idcs` is the dictionary that maps each `VarName` instance to its index in - `md.vns`, `md.ranges`, `md.dists`, `md.orders` and `md.flags`. -- `md.vns[md.idcs[vn]] == vn`. -- `md.dists[md.idcs[vn]]` is the distribution of `vn`. -- `md.gids[md.idcs[vn]]` is the set of algorithms used to sample `vn`. This is used in - the Gibbs sampling process. -- `md.orders[md.idcs[vn]]` is the number of `observe` statements before `vn` is sampled. -- `md.ranges[md.idcs[vn]]` is the index range of `vn` in `md.vals`. -- `md.vals[md.ranges[md.idcs[vn]]]` is the linearized vector of values of corresponding to `vn`. -- `md.flags` is a dictionary of true/false flags. `md.flags[flag][md.idcs[vn]]` is the - value of `flag` corresponding to `vn`. - -Note that in order to make `md::Metadata` type stable, all the `md.vns` must have the same symbol and distribution type. However, one can have a single Julia variable, e.g. `x`, that is a matrix or a hierarchical array sampled in partitions, e.g. `x[1][:] ~ MvNormal(zeros(2), 1.0); x[2][:] ~ MvNormal(ones(2), 1.0)`. The symbol `x` can still be managed by a single `md::Metadata` without hurting the type stability since all the distributions on the RHS of `~` are of the same type. - -However, in `Turing` models one cannot have this restriction, so we must use a type unstable `Metadata` if we want to use one `Metadata` instance for the whole model. This is what `UntypedVarInfo` does. A type unstable `Metadata` will still work but will have inferior performance. - -To strike a balance between flexibility and performance when constructing the `spl::Sampler` instance, the model is first run by sampling the parameters in `P` from their priors using an `UntypedVarInfo`, i.e. a type unstable `Metadata` is used for all the variables. Then once all the symbols and distribution types have been identified, a `vi::TypedVarInfo` is constructed where `vi.metadata` is a `NamedTuple` mapping each symbol in `P` to a specialized instance of `Metadata`. So as long as each symbol in `P` is sampled from only one type of distributions, `vi::TypedVarInfo` will have fully concretely typed fields which brings out the peak performance of Julia. diff --git a/src/for-developers/how_turing_implements_abstractmcmc.md b/src/for-developers/how_turing_implements_abstractmcmc.md deleted file mode 100644 index de0048688..000000000 --- a/src/for-developers/how_turing_implements_abstractmcmc.md +++ /dev/null @@ -1,239 +0,0 @@ ---- -title: How Turing implements AbstractMCMC ---- - -# How Turing implements AbstractMCMC - -Prerequisite: [Interface guide](https://turing.ml/dev/docs/for-developers/interface). - -## Introduction - -Consider the following Turing, code block: - -```julia -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -mod = gdemo(1.5, 2) -alg = IS() -n_samples = 1000 - -chn = sample(mod, alg, n_samples) -``` - -The function `sample` is part of the AbstractMCMC interface. As explained in the [interface guide](https://turing.ml/dev/docs/for-developers/interface), building a a sampling method that can be used by `sample` consists in overloading the structs and functions in `AbstractMCMC`. The interface guide also gives a standalone example of their implementation, [`AdvancedMH.jl`](). - -Turing sampling methods (most of which are written [here](https://github.com/TuringLang/Turing.jl/tree/master/src/inference)) also implement `AbstractMCMC`. Turing defines a particular architecture for `AbstractMCMC` implementations, that enables working with models defined by the `@model` macro, and uses DynamicPPL as a backend. The goal of this page is to describe this architecture, and how you would go about implementing your own sampling method in Turing, using Importance Sampling as an example. I don't go into all the details: for instance, I don't address selectors or parallelism. - -First, we explain how Importance Sampling works in the abstract. Consider the model defined in the first code block. Mathematically, it can be written: - -$\begin{align} -s &\sim \text{InverseGamma}(2, 3) \\ -m &\sim \text{Normal}(0, \sqrt{s}) \\ -x &\sim \text{Normal}(m, \sqrt{s}) \\ -y &\sim \text{Normal}(m, \sqrt{s}) -\end{align}$ - -The **latent** variables are \$\$s\$\$ and \$\$m\$\$, the **observed** variables are \$\$x\$\$ and \$\$y\$\$. The model **joint** distribution \$\$p(s,m,x,y)\$\$ decomposes into the **prior** \$\$p(s,m)\$\$ and the **likelihood** \$\$p(x,y \mid s,m)\$\$. Since \$\$x = 1.5\$\$ and \$\$y = 2\$\$ are observed, the goal is to infer the **posterior** distribution \$\$p(s,m \mid x,y)\$\$. - -Importance Sampling produces independent samples \$\$(s\_i, m\_i)\$\$ from the prior distribution. It also outputs unnormalized weights \$\$w\_i = \frac {p(x,y,s\_i,m\_i)} {p(s\_i, m\_i)} = p(x,y \mid s\_i, m\_i)\$\$ such that the empirical distribution \$\$\frac 1 N \sum\limits\_{i =1}^N \frac {w_i} {\sum\limits_{j=1}^N w\_j} \delta_{(s\_i, m\_i)}\$\$ is a good approximation of the posterior. - -## 1. Define a `Sampler` - -Recall the last line of the above code block: - -```julia -chn = sample(mod, alg, n_samples) -``` - -Here `sample` takes as arguments a **model** `mod`, an **algorithm** `alg`, and a **number of samples** `n_samples`, and returns an instance `chn` of `Chains` which can be analysed using the functions in `MCMCChains`. - -### Models - -To define a **model**, you declare a joint distribution on variables in the `@model` macro, and specify which variables are observed and which should be inferred, as well as the value of the observed variables. Thus, when implementing Importance Sampling, - -```julia -mod = gdemo(1.5, 2) -``` - -creates an instance `mod` of the struct `Model`, which corresponds to the observations of a value of `1.5` for `x`, and a value of `2` for `y`. - -This is all handled by DynamicPPL, more specifically [here](https://github.com/TuringLang/DynamicPPL.jl/blob/master/src/model.jl). I will return to how models are used to inform sampling algorithms [below](#assumeobserve). - -### Algorithms - -An **algorithm** is just a sampling method: in Turing, it is a subtype of the abstract type `InferenceAlgorithm`. Defining an algorithm may require specifying a few high-level parameters. For example, "Hamiltonian Monte-Carlo" may be too vague, but "Hamiltonian Monte Carlo with 10 leapfrog steps per proposal and a stepsize of 0.01" is an algorithm. "Metropolis-Hastings" may be too vague, but "Metropolis-Hastings with proposal distribution `p`" is an algorithm. \$\$\epsilon\$\$ - -Thus - -```julia -stepsize = 0.01 -L = 10 -alg = HMC(stepsize, L) -``` - -defines a Hamiltonian Monte-Carlo algorithm, an instance of `HMC`, which is a subtype of `InferenceAlgorithm`. - -In the case of Importance Sampling, there is no need to specify additional parameters: - -```julia -alg = IS() -``` - -defines an Importance Sampling algorithm, an instance of `IS` which is a subtype of `InferenceAlgorithm`. - -When creating your own Turing sampling method, you must therefore build a subtype of `InferenceAlgorithm` corresponding to your method. - -### Samplers - -Samplers are **not** the same as algorithms. An algorithm is a generic sampling method, a sampler is an object that stores information about how algorithm and model interact during sampling, and is modified as sampling progresses. The `Sampler` struct is defined in DynamicPPL. - -Turing implements `AbstractMCMC`'s `AbstractSampler` with the `Sampler` struct defined in `DynamicPPL`. The most important attributes of an instance `spl` of `Sampler` are: - -* `spl.alg`: the sampling method used, an instance of a subtype of `InferenceAlgorithm` -* `spl.state`: information about the sampling process, see [below](#States) - -When you call `sample(mod, alg, n_samples)`, Turing first uses `model` and `alg` to build an instance `spl` of `Sampler` , then calls the native `AbstractMCMC` function `sample(mod, spl, n_samples)`. - -When you define your own Turing sampling method, you must therefore build: - -* a **sampler constructor** that uses a model and an algorithm to initialize an instance of `Sampler`. For Importance Sampling: - -```julia -function Sampler(alg::IS, model::Model, s::Selector) - info = Dict{Symbol, Any}() - state = ISState(model) - return Sampler(alg, info, s, state) -end -``` - -* a **state** struct implementing `AbstractSamplerState` corresponding to your method: we cover this in the following paragraph. - -### States - -The `vi` field contains all the important information about sampling: first and foremost, the values of all the samples, but also the distributions from which they are sampled, the names of model parameters, and other metadata. As we will see below, many important steps during sampling correspond to queries or updates to `spl.state.vi`. - -By default, you can use `SamplerState`, a concrete type defined in `inference/Inference.jl`, which extends `AbstractSamplerState` and has no field except for `vi`: - -```julia -mutable struct SamplerState{VIType<:VarInfo} <: AbstractSamplerState - vi :: VIType -end -``` - -When doing Importance Sampling, we care not only about the values of the samples but also their weights. We will see below that the weight of each sample is also added to `spl.state.vi`. Moreover, the average \$\$\frac 1 N \sum\limits\_{j=1}^N w\_i = \frac 1 N \sum\limits\_{j=1}^N p(x,y \mid s\_i, m\_i)\$\$ of the sample weights is a particularly important quantity: - -* it is used to **normalize** the **empirical approximation** of the posterior distribution -* its logarithm is the importance sampling **estimate** of the **log evidence** \$\$\log p(x, y)\$\$ - -To avoid having to compute it over and over again, `is.jl`defines an IS-specific concrete type `ISState` for sampler states, with an additional field `final_logevidence` containing \$\$\log \left( \frac 1 N \sum\limits\_{j=1}^N w\_i \right)\$\$. - -```julia -mutable struct ISState{V<:VarInfo, F<:AbstractFloat} <: AbstractSamplerState - vi :: V - final_logevidence :: F -end - -# additional constructor -ISState(model::Model) = ISState(VarInfo(model), 0.0) -``` - -The following diagram summarizes the hierarchy presented above. - -![hierarchy](how_turing_implements_abstractmcmc_files/hierarchy.png) - -## 2. Overload the functions used inside `mcmcsample` - -A lot of the things here are method-specific. However Turing also has some functions that make it easier for you to implement these functions, for examples . - -### Transitions - -`AbstractMCMC` stores information corresponding to each individual sample in objects called `transition`, but does not specify what the structure of these objects could be. You could decide to implement a type `MyTransition` for transitions corresponding to the specifics of your methods. However, there are many situations in which the only information you need for each sample is: - -* its value: \$\$\theta\$\$ -* log of the joint probability of the observed data and this sample: `lp` - -`Inference.jl` [defines](https://github.com/TuringLang/Turing.jl/blob/master/src/inference/Inference.jl#L103) a struct `Transition`, which corresponds to this default situation - -```julia -struct Transition{T, F<:AbstractFloat} - θ :: T - lp :: F -end -``` - -It also [contains](https://github.com/TuringLang/Turing.jl/blob/master/src/inference/Inference.jl#L108) a constructor that builds an instance of `Transition` from an instance `spl` of `Sampler`: \$\$\theta\$\$ is `spl.state.vi` converted to a `namedtuple`, and `lp` is `getlogp(spl.state.vi)`. `is.jl` uses this default constructor at the end of the `step!` function [here](https://github.com/TuringLang/Turing.jl/blob/master/src/inference/is.jl#L58). - -### How `sample` works - -A crude summary, which ignores things like parallelism, is the following: - - `sample` calls `mcmcsample`, which calls - -* `sample_init!` to set things up -* `step!` repeatedly to produce multiple new transitions -* `sample_end!` to perform operations once all samples have been obtained -* `bundle_samples` to convert a vector of transitions into a more palatable type, for instance a `Chain`. - -You can of course implement all of these functions, but `AbstractMCMC` as well as Turing also provide default implementations for simple cases. For instance, importance sampling uses the default implementations of `sample_init!` and `bundle_samples`, which is why you don't see code for them inside `is.jl`. - -## 3. Overload `assume` and `observe` - -The functions mentioned above, such as `sample_init!`, `step!`, etc., must of course use information about the model in order to generate samples! In particular, these functions may need **samples from distributions** defined in the model, or to **evaluate the density of these distributions** at some values of the corresponding parameters or observations. - -For an example of the former, consider **Importance Sampling** as defined in `is.jl`. This implementation of Importance Sampling uses the model prior distribution as a proposal distribution, and therefore requires **samples from the prior distribution** of the model. Another example is **Approximate Bayesian Computation**, which requires multiple **samples from the model prior and likelihood distributions** in order to generate a single sample. - -An example of the latter is the **Metropolis-Hastings** algorithm. At every step of sampling from a target posterior \$\$p(\theta \mid x\_{\text{obs}})\$\$, in order to compute the acceptance ratio, you need to **evaluate the model joint density** \$\$p(\theta\_{\text{prop}}, x\_{\text{obs}})\$\$ with \$\$\theta\_{\text{prop}}\$\$ a sample from the proposal and \$\$x\_{\text{obs}}\$\$ the observed data. - -This begs the question: how can these functions access model information during sampling? Recall that the model is stored as an instance `m` of `Model`. One of the attributes of `m` is the model evaluation function `m.f`, which is built by compiling the `@model` macro. Executing `f` runs the tilde statements of the model in order, and adds model information to the sampler (the instance of `Sampler` that stores information about the ongoing sampling process) at each step (see [here](https://turing.ml/dev/docs/for-developers/compiler) for more information about how the `@model` macro is compiled). The DynamicPPL functions `assume` and `observe` determine what kind of information to add to the sampler for every tilde statement. - -Consider an instance `m` of `Model` and a sampler `spl`, with associated `VarInfo` `vi = spl.state.vi`. At some point during the sampling process, an AbstractMCMC function such as `step!` calls `m(vi, ...)`, which calls the model evaluation function `m.f(vi, ...)`. - -* for every tilde statement in the `@model` macro, `m.f(vi, ...)` returns model-related information (samples, value of the model density, etc.), and adds it to `vi`. How does it do that? - * recall that the code for `m.f(vi, ...)` is automatically generated by compilation of the `@model` macro - * for every tilde statement in the `@model` declaration, this code contains a call to `assume(vi, ...)` if the variable on the LHS of the tilde is a **model parameter to infer**, and `observe(vi, ...)` if the variable on the LHS of the tilde is an **observation** - * in the file corresponding to your sampling method (ie in `Turing.jl/src/inference/.jl`), you have **overloaded** `assume` and `observe`, so that they can modify `vi` to include the information and samples that you care about! - * at a minimum, `assume` and `observe` return the log density `lp` of the sample or observation. the model evaluation function then immediately calls `acclogp!(vi, lp)`, which adds `lp` to the value of the log joint density stored in `vi`. - -Here's what `assume` looks like for Importance Sampling: - -```julia -function DynamicPPL.assume(rng, spl::Sampler{<:IS}, dist::Distribution, vn::VarName, vi) - r = rand(rng, dist) - push!(vi, vn, r, dist, spl) - return r, 0 -end -``` - -The function first generates a sample `r` from the distribution `dist` (the right hand side of the tilde statement). It then adds `r` to `vi`, and returns `r` and 0. - -The `observe` function is even simpler: - -```julia -function DynamicPPL.observe(spl::Sampler{<:IS}, dist::Distribution, value, vi) - return logpdf(dist, value) -end -``` - -It simply returns the density (in the discrete case, the probability) of the observed value under the distribution `dist`. - -## 4. Summary: Importance Sampling step by step - -We focus on the AbstractMCMC functions that are overriden in `is.jl` and executed inside `mcmcsample`: `step!`, which is called `n_samples` times, and `sample_end!`, which is executed once after those `n_samples` iterations. - -* During the \$\$i\$\$-th iteration, `step!` does 3 things: - * `empty!(spl.state.vi)`: remove information about the previous sample from the sampler's `VarInfo` - * `model(rng, spl.state.vi, spl)`: call the model evaluation function - * calls to `assume` add the samples from the prior \$\$s\_i\$\$ and \$\$m\_i\$\$ to `spl.state.vi` - * calls to both `assume` or `observe` are followed by the line `acclogp!(vi, lp)`, where `lp` is an output of `assume` and `observe` - * `lp` is set to 0 after `assume`, and to the value of the density at the observation after `observe` - * when all the tilde statements have been covered, `spl.state.vi.logp[]` is the sum of the `lp`, ie the likelihood \$\$\log p(x, y \mid s\_i, m\_i) = \log p(x \mid s\_i, m\_i) + \log p(y \mid s\_i, m\_i)\$\$ of the observations given the latent variable samples \$\$s\_i\$\$ and \$\$m\_i\$\$. - * `return Transition(spl)`: build a transition from the sampler, and return that transition - * the transition's `vi` field is simply `spl.state.vi` - * the `lp` field contains the likelihood `spl.state.vi.logp[]` -* When the, `n_samples` iterations are completed, `sample_end!` fills the `final_logevidence` field of `spl.state` - * it simply takes the logarithm of the average of the sample weights, using the log weights for numerical stability \ No newline at end of file diff --git a/src/for-developers/how_turing_implements_abstractmcmc_files/hierarchy.png b/src/for-developers/how_turing_implements_abstractmcmc_files/hierarchy.png deleted file mode 100644 index 1abeed687..000000000 Binary files a/src/for-developers/how_turing_implements_abstractmcmc_files/hierarchy.png and /dev/null differ diff --git a/src/for-developers/interface.md b/src/for-developers/interface.md deleted file mode 100644 index f8ed9507b..000000000 --- a/src/for-developers/interface.md +++ /dev/null @@ -1,365 +0,0 @@ ---- -title: Interface Guide -toc: true ---- - -# The sampling interface - -Turing implements a sampling interface (hosted at -[AbstractMCMC](https://github.com/TuringLang/AbstractMCMC.jl)) that is intended to provide -a common framework for Markov chain Monte Carlo samplers. The interface presents several -structures and functions that one needs to overload in order to implement an -interface-compatible sampler. - -This guide will demonstrate how to implement the interface without Turing. - -## Interface overview - -Any implementation of an inference method that uses the AbstractMCMC interface should -implement a subset of the following types and functions: - -1. A subtype of `AbstractSampler`, defined as a mutable struct containing state information or sampler parameters. -2. A function `sample_init!` which performs any necessary set-up (default: do not perform any set-up). -3. A function `step!` which returns a transition that represents a single draw from the sampler. -4. A function `transitions_init` which returns a container for the transitions obtained from the sampler - (default: return a `Vector{T}` of length `N` where `T` is the type of the transition obtained in the first step and `N` is the number of requested samples). -5. A function `transitions_save!` which saves transitions to the container (default: save the transition of iteration `i` - at position `i` in the vector of transitions). -6. A function `sample_end!` which handles any sampler wrap-up (default: do not perform any wrap-up). -7. A function `bundle_samples` which accepts the container of transitions and returns a collection of samples - (default: return the vector of transitions). - -The interface methods with exclamation points are those that are intended to allow for -state mutation. Any mutating function is meant to allow mutation where needed -- you might -use: - -- `sample_init!` to run some kind of sampler preparation, before sampling begins. This - could mutate a sampler's state. -- `step!` might mutate a sampler flag after each sample. -- `sample_end!` contains any wrap-up you might need to do. If you were sampling in a - transformed space, this might be where you convert everything back to a constrained space. - -## Why do you have an interface? - -The motivation for the interface is to allow Julia's fantastic probabilistic programming -language community to have a set of standards and common implementations so we can all -thrive together. Markov chain Monte Carlo methods tend to have a very similar framework to -one another, and so a common interface should help more great inference methods built in -single-purpose packages to experience more use among the community. - -## Implementing Metropolis-Hastings without Turing - -[Metropolis-Hastings](https://en.wikipedia.org/wiki/Markov_chain_Monte_Carlo) is often the -first sampling method that people are exposed to. It is a very straightforward algorithm -and is accordingly the easiest to implement, so it makes for a good example. In this -section, you will learn how to use the types and functions listed above to implement the -Metropolis-Hastings sampler using the MCMC interface. - -The full code for this implementation is housed in -[AdvancedMH.jl](https://github.com/TuringLang/AdvancedMH.jl). - -### Imports - -Let's begin by importing the relevant libraries. We'll import `AbstracMCMC`, which contains -the interface framework we'll fill out. We also need `Distributions` and `Random`. - -```julia -# Import the relevant libraries. -import AbstractMCMC -using Distributions -using Random -``` - -An interface extension (like the one we're writing right now) typically requires that you overload or implement several functions. Specifically, you should `import` the functions you intend to overload. This next code block accomplishes that. - -From `Distributions`, we need `Sampleable`, `VariateForm`, and `ValueSupport`, three abstract types that define a distribution. Models in the interface are assumed to be subtypes of `Sampleable{VariateForm, ValueSupport}`. In this section our model is going be be extremely simple, so we will not end up using these except to make sure that the inference functions are dispatching correctly. - -### Sampler - -Let's begin our sampler definition by defining a sampler called `MetropolisHastings` which -is a subtype of `AbstractSampler`. Correct typing is very important for proper interface -implementation -- if you are missing a subtype, your method may not be dispatched to when -you call `sample`. - -```julia -# Define a sampler type. -struct MetropolisHastings{T, D} <: AbstractMCMC.AbstractSampler - init_θ::T - proposal::D -end - -# Default constructors. -MetropolisHastings(init_θ::Real) = MetropolisHastings(init_θ, Normal(0,1)) -MetropolisHastings(init_θ::Vector{<:Real}) = MetropolisHastings(init_θ, MvNormal(length(init_θ),1)) -``` - -Above, we have defined a sampler that stores the initial parameterization of the prior, -and a distribution object from which proposals are drawn. You can have a struct that has no -fields, and simply use it for dispatching onto the relevant functions, or you can store a -large amount of state information in your sampler. - -The general intuition for what to store in your sampler struct is that anything you may -need to perform inference between samples but you don't want to store in a transition -should go into the sampler struct. It's the only way you can carry non-sample related state -information between `step!` calls. - -### Model - -Next, we need to have a model of some kind. A model is a struct that's a subtype of -`AbstractModel` that contains whatever information is necessary to perform inference on -your problem. In our case we want to know the mean and variance parameters for a standard -Normal distribution, so we can keep our model to the log density of a Normal. - -Note that we only have to do this because we are not yet integrating the sampler with Turing --- Turing has a very sophisticated modelling engine that removes the need to define custom -model structs. - -```julia -# Define a model type. Stores the log density function. -struct DensityModel{F<:Function} <: AbstractMCMC.AbstractModel - ℓπ::F -end -``` - -### Transition - -The next step is to define some transition which we will return from each `step!` call. -We'll keep it simple by just defining a wrapper struct that contains the parameter draws -and the log density of that draw: - -```julia -# Create a very basic Transition type, only stores the -# parameter draws and the log probability of the draw. -struct Transition{T, L} - θ::T - lp::L -end - -# Store the new draw and its log density. -Transition(model::DensityModel, θ) = Transition(θ, ℓπ(model, θ)) -``` - -`Transition` can now store any type of parameter, whether it's a vector of draws from -multiple parameters or a single univariate draw. - -### Metropolis-Hastings - -Now it's time to get into the actual inference. We've defined all of the core pieces we -need, but we need to implement the `step!` function which actually performs inference. - -As a refresher, Metropolis-Hastings implements a very basic algorithm: - -1. Pick some initial state, \$\$\theta\_0\$\$. -2. For \$\$t\$\$ in \$\$[1,N]\$\$, do - - a. Generate a proposal parameterization \$\$θ'\_t \sim q(\theta'\_t \mid \theta\_{t-1})\$\$. - - b. Calculate the acceptance probability, \$\$\alpha = \text{min}\Big[1,\frac{\pi(θ'\_t)}{\pi(\theta\_{t-1})} \frac{q(θ\_{t-1} \mid θ'\_t)}{q(θ'\_t \mid θ\_{t-1})}) \Big]\$\$. - - c. If \$\$U \le α\$\$ where \$\$U \sim [0,1]\$\$, then \$\$\theta\_t = \theta'\_t\$\$. Otherwise, \$\$\theta\_t = \theta\_{t-1}\$\$. - -Of course, it's much easier to do this in the log space, so the acceptance probability is -more commonly written as - -$$\alpha = \min\Big[\log \pi(θ'\_t) - \log \pi(θ\_{t-1}) + \log q(θ\_{t-1} \mid θ'\_t) - \log q(θ'\_t \mid θ\_{t-1}), 0\Big]$$ - -In interface terms, we should do the following: - -1. Make a new transition containing a proposed sample. -2. Calculate the acceptance probability. -3. If we accept, return the new transition, otherwise, return the old one. - -### Steps - -The `step!` function is the function that performs the bulk of your inference. In our case, -we will implement two `step!` functions -- one for the very first iteration, and one for -every subsequent iteration. - -```julia -# Define the first step! function, which is called at the -# beginning of sampling. Return the initial parameter used -# to define the sampler. -function AbstractMCMC.step!( - rng::AbstractRNG, - model::DensityModel, - spl::MetropolisHastings, - N::Integer, - ::Nothing; - kwargs... -) - return Transition(model, spl.init_θ) -end -``` - -The first `step!` function just packages up the initial parameterization inside the -sampler, and returns it. We implicity accept the very first parameterization. - -The other `step!` function performs the usual steps from Metropolis-Hastings. Included are -several helper functions, `proposal` and `q`, which are designed to replicate the functions -in the pseudocode above. - -- `proposal` generates a new proposal in the form of a `Transition`, which can be - univariate if the value passed in is univariate, or it can be multivariate if the - `Transition` given is multivariate. Proposals use a basic `Normal` or `MvNormal` proposal - distribution. -- `q` returns the log density of one parameterization conditional on another, according to - the proposal distribution. -- `step!` generates a new proposal, checks the acceptance probability, and then returns - either the previous transition or the proposed transition. - -```julia -# Define a function that makes a basic proposal depending on a univariate -# parameterization or a multivariate parameterization. -propose(spl::MetropolisHastings, model::DensityModel, θ::Real) = - Transition(model, θ + rand(spl.proposal)) -propose(spl::MetropolisHastings, model::DensityModel, θ::Vector{<:Real}) = - Transition(model, θ + rand(spl.proposal)) -propose(spl::MetropolisHastings, model::DensityModel, t::Transition) = - propose(spl, model, t.θ) - -# Calculates the probability `q(θ|θcond)`, using the proposal distribution `spl.proposal`. -q(spl::MetropolisHastings, θ::Real, θcond::Real) = logpdf(spl.proposal, θ - θcond) -q(spl::MetropolisHastings, θ::Vector{<:Real}, θcond::Vector{<:Real}) = - logpdf(spl.proposal, θ - θcond) -q(spl::MetropolisHastings, t1::Transition, t2::Transition) = q(spl, t1.θ, t2.θ) - -# Calculate the density of the model given some parameterization. -ℓπ(model::DensityModel, θ) = model.ℓπ(θ) -ℓπ(model::DensityModel, t::Transition) = t.lp - -# Define the other step function. Returns a Transition containing -# either a new proposal (if accepted) or the previous proposal -# (if not accepted). -function AbstractMCMC.step!( - rng::AbstractRNG, - model::DensityModel, - spl::MetropolisHastings, - ::Integer, - θ_prev::Transition; - kwargs... -) - # Generate a new proposal. - θ = propose(spl, model, θ_prev) - - # Calculate the log acceptance probability. - α = ℓπ(model, θ) - ℓπ(model, θ_prev) + q(spl, θ_prev, θ) - q(spl, θ, θ_prev) - - # Decide whether to return the previous θ or the new one. - if log(rand(rng)) < min(α, 0.0) - return θ - else - return θ_prev - end -end -``` - -### Chains - -In the default implementation, `sample` just returns a vector of all transitions. If -instead you would like to obtain a `Chains` object (e.g., to simplify downstream analysis), -you have to implement the `bundle_samples` function as well. It accepts the vector of -transitions and returns a collection of samples. Fortunately, our `Transition` is -incredibly simple, and we only need to build a little bit of functionality to accept custom -parameter names passed in by the user. - -```julia -# A basic chains constructor that works with the Transition struct we defined. -function AbstractMCMC.bundle_samples( - rng::AbstractRNG, - ℓ::DensityModel, - s::MetropolisHastings, - N::Integer, - ts::Vector{<:Transition}, - chain_type::Type{Any}; - param_names=missing, - kwargs... -) - # Turn all the transitions into a vector-of-vectors. - vals = copy(reduce(hcat,[vcat(t.θ, t.lp) for t in ts])') - - # Check if we received any parameter names. - if ismissing(param_names) - param_names = ["Parameter $i" for i in 1:(length(first(vals))-1)] - end - - # Add the log density field to the parameter names. - push!(param_names, "lp") - - # Bundle everything up and return a Chains struct. - return Chains(vals, param_names, (internals=["lp"],)) -end -``` - -All done! - -You can even implement different output formats by implementing `bundle_samples` for -different `chain_type`s, which can be provided as keyword argument to `sample`. As default -`sample` uses `chain_type = Any`. - -### Testing the implementation - -Now that we have all the pieces, we should test the implementation by defining a model to -calculate the mean and variance parameters of a Normal distribution. We can do this by -constructing a target density function, providing a sample of data, and then running the -sampler with `sample`. - -```julia -# Generate a set of data from the posterior we want to estimate. -data = rand(Normal(5, 3), 30) - -# Define the components of a basic model. -insupport(θ) = θ[2] >= 0 -dist(θ) = Normal(θ[1], θ[2]) -density(θ) = insupport(θ) ? sum(logpdf.(dist(θ), data)) : -Inf - -# Construct a DensityModel. -model = DensityModel(density) - -# Set up our sampler with initial parameters. -spl = MetropolisHastings([0.0, 0.0]) - -# Sample from the posterior. -chain = sample(model, spl, 100000; param_names=["μ", "σ"]) -``` - -If all the interface functions have been extended properly, you should get an output from -`display(chain)` that looks something like this: - -``` -Object of type Chains, with data of type 100000×3×1 Array{Float64,3} - -Iterations = 1:100000 -Thinning interval = 1 -Chains = 1 -Samples per chain = 100000 -internals = lp -parameters = μ, σ - -2-element Array{ChainDataFrame,1} - -Summary Statistics - -│ Row │ parameters │ mean │ std │ naive_se │ mcse │ ess │ r_hat │ -│ │ Symbol │ Float64 │ Float64 │ Float64 │ Float64 │ Any │ Any │ -├─────┼────────────┼─────────┼──────────┼────────────┼────────────┼─────────┼─────────┤ -│ 1 │ μ │ 5.33157 │ 0.854193 │ 0.0027012 │ 0.00893069 │ 8344.75 │ 1.00009 │ -│ 2 │ σ │ 4.54992 │ 0.632916 │ 0.00200146 │ 0.00534942 │ 14260.8 │ 1.00005 │ - -Quantiles - -│ Row │ parameters │ 2.5% │ 25.0% │ 50.0% │ 75.0% │ 97.5% │ -│ │ Symbol │ Float64 │ Float64 │ Float64 │ Float64 │ Float64 │ -├─────┼────────────┼─────────┼─────────┼─────────┼─────────┼─────────┤ -│ 1 │ μ │ 3.6595 │ 4.77754 │ 5.33182 │ 5.89509 │ 6.99651 │ -│ 2 │ σ │ 3.5097 │ 4.09732 │ 4.47805 │ 4.93094 │ 5.96821 │ -``` - -It looks like we're extremely close to our true parameters of `Normal(5,3)`, though with a -fairly high variance due to the low sample size. - -## Conclusion - -We've seen how to implement the sampling interface for general projects. Turing's interface -methods are ever-evolving, so please open an issue at -[AbstractMCMC](https://github.com/TuringLang/AbstractMCMC.jl) with feature requests or -problems. diff --git a/src/for-developers/variational_inference.md b/src/for-developers/variational_inference.md deleted file mode 100644 index b30148c3b..000000000 --- a/src/for-developers/variational_inference.md +++ /dev/null @@ -1,312 +0,0 @@ ---- -title: Variational Inference -toc: true ---- - -# Overview - -In this post we'll have a look at what's know as **variational inference (VI)**, a family of _approximate_ Bayesian inference methods. In particular, we will focus on one of the more standard VI methods called **Automatic Differentation Variational Inference (ADVI)**. If - -Here we'll have a look at the theory behind VI, but if you're interested in how to use ADVI in Turing.jl, [checkout this tutorial](../../tutorials/9-variationalinference). - -# Motivation - -In Bayesian inference one usually specifies a model as follows: given data \$\$\\{ x\_i\\}\_{i = 1}^n\$\$, - -\$\$ -\\begin{align*} - \text\{prior: \} \quad z &\sim p(z) \\\\ - \text\{likelihood:\} \\quad x_i &\overset{\text{i.i.d.}}{\sim} p(x \mid z) \quad \text{where} \quad i = 1, \dots, n -\\end{align*} -\$\$ - -where \$\$\overset{\text{i.i.d.}}{\sim}\$\$ denotes that the samples are identically independently distributed. Our goal in Bayesian inference is then to find the _posterior_ -\$\$ -p(z \mid \\{ x\_i \\}\_{i = 1}^n) = \prod\_{i=1}^{n} p(z \mid x\_i) -\$\$ -In general one cannot obtain a closed form expression for \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$, but one might still be able to _sample_ from \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ with guarantees of converging to the target posterior \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ as the number of samples go to \$\$\infty\$\$, e.g. MCMC. - -As you are hopefully already aware, Turing.jl provides a lot of different methods with asymptotic exactness guarantees that we can apply to such a problem! - -Unfortunately, these unbiased samplers can be prohibitively expensive to run. As the model \$\$p\$\$ increases in complexity, the convergence of these unbiased samplers can slow down dramatically. Still, in the _infinite_ limit, these methods should converge to the true posterior! But infinity is fairly large, like, _at least_ more than 12, so this might take a while. - -In such a case it might be desirable to sacrifice some of these asymptotic guarantees, and instead _approximate_ the posterior \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ using some other model which we'll denote \$\$q(z)\$\$. - -There are multiple approaches to take in this case, one of which is **variational inference (VI)**. - -# Variational Inference (VI) - -In VI, we're looking to approximate \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n )\$\$ using some _approximate_ or _variational_ posterior \$\$q(z)\$\$. - -To approximate something you need a notion of what "close" means. In the context of probability densities a standard such "measure" of closeness is the _Kullback-Leibler (KL) divergence_ , though this is far from the only one. The KL-divergence is defined between two densities \$\$q(z)\$\$ and \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ as - -\$\$ -\begin{align*} - \mathrm{D\_{KL}} \big( q(z), p(z \mid \\{ x\_i \\}\_{i = 1}^n) \big) &= \int \log \bigg( \frac{q(z)}{\prod\_{i = 1}^n p(z \mid x\_i)} \bigg) q(z) \mathrm{d}{z} \\\\ - &= \mathbb{E}\_{z \sim q(z)} \big[ \log q(z) - \sum\_{i = 1}^n \log p(z \mid x\_i) \big] \\\\ - &= \mathbb{E}\_{z \sim q(z)} \big[ \log q(z) \big] - \sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(z \mid x\_i) \big] -\end{align*} -\$\$ - - -It's worth noting that unfortunately the KL-divergence is _not_ a metric/distance in the analysis-sense due to its lack of symmetry. On the other hand, it turns out that minimizing the KL-divergence that it's actually equivalent to maximizing the log-likelihood! Also, under reasonable restrictions on the densities at hand, - -\$\$ -\mathrm{D\_{KL}}\big(q(z), p(z \mid \\{ x\_i \\}\_{i = 1}^n) \big) = 0 \quad \iff \quad q(z) = p(z \mid \\{ x\_i \\}\_{i = 1}^n), \quad \forall z -\$\$ - -Therefore one could (and we will) attempt to approximate \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ using a density \$\$q(z)\$\$ by minimizing the KL-divergence between these two! - -One can also show that \$\$\mathrm{D\_{KL}} \ge 0\$\$, which we'll need later. Finally notice that the KL-divergence is only well-defined when in fact \$\$q(z)\$\$ is zero everywhere \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ is zero, i.e. - -\$\$ -\mathrm{supp}\big(q(z)\big) \subseteq \mathrm{supp}\big(p(z \mid x)\big) -\$\$ - -Otherwise there might be a point \$\$z\_0 \sim q(z)\$\$ such that \$\$p(z\_0 \mid \\{ x\_i \\}\_{i = 1}^n) = 0\$\$, resulting in \$\$\log\big(\frac{q(z)}{0}\big)\$\$ which doesn't make sense! - -One major problem: as we can see in the definition of the KL-divergence, we need \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ for any \$\$z\$\$ if we want to compute the KL-divergence between this and \$\$q(z)\$\$. We don't have that. The entire reason we even do Bayesian inference is that we don't know the posterior! Cleary this isn't going to work. _Or is it?!_ - -## Computing KL-divergence without knowing the posterior - -First off, recall that - -\$\$ -p(z \mid x\_i) = \frac{p(x\_i, z)}{p(x\_i)} -\$\$ - -so we can write - -\$\$ -\begin{align*} -\mathrm{D\_{KL}} \big( q(z), p(z \mid \\{ x\_i \\}\_{i = 1}^n) \big) &= \mathbb{E}\_{z \sim q(z)} \big[ \log q(z) \big] - \sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(x\_i, z) - \log p(x\_i) \big] \\\\ - &= \mathbb{E}\_{z \sim q(z)} \big[ \log q(z) \big] - \sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(x\_i, z) \big] + \sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(x\_i) \big] \\\\ - &= \mathbb{E}\_{z \sim q(z)} \big[ \log q(z) \big] - \sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(x\_i, z) \big] + \sum\_{i = 1}^n \log p(x\_i) -\end{align*} -\$\$ - -where in the last equality we used the fact that \$\$p(x\_i)\$\$ is independent of \$\$z\$\$. - -Now you're probably thinking "Oh great! Now you've introduced \$\$p(x\_i)\$\$ which we _also_ can't compute (in general)!". Woah. Calm down human. Let's do some more algebra. The above expression can be rearranged to - -\$\$ -\mathrm{D\_{KL}} \big( q(z), p(z \mid \\{ x\_i \\}\_{i = 1}^n) \big) + \underbrace{\sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(x\_i, z) \big] - \mathbb{E}\_{z \sim q(z)} \big[ \log q(z) \big]}\_{=: \mathrm{ELBO}(q)} = \underbrace{\sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(x\_i) \big]}\_{\text{constant}} -\$\$ - -See? The left-hand side is _constant_ and, as we mentioned before, \$\$\mathrm{D\_{KL}} \ge 0\$\$. What happens if we try to _maximize_ the term we just gave the completely arbitrary name \$\$\mathrm{ELBO}\$\$? Well, if \$\$\mathrm{ELBO}\$\$ goes up while \$\$p(x\_i)\$\$ stays constant then \$\$\mathrm{D\_{KL}}\$\$ _has to_ go down! That is, the \$\$q(z)\$\$ which _minimizes_ the KL-divergence is the same \$\$q(z)\$\$ which _maximizes_ \$\$\mathrm{ELBO}(q)\$\$: - -\$\$ -\underset{q}{\mathrm{argmin}} \ \mathrm{D\_{KL}} \big( q(z), p(z \mid \\{ x\_i \\}\_{i = 1}^n) \big) = \underset{q}{\mathrm{argmax}} \ \mathrm{ELBO}(q) -\$\$ - -where - -\$\$ -\begin{align*} -\mathrm{ELBO}(q) &:= \bigg( \sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(x\_i, z) \big] \bigg) - \mathbb{E}\_{z \sim q(z)} \big[ \log q(z) \big] \\\\ - &= \bigg( \sum\_{i = 1}^n \mathbb{E}\_{z \sim q(z)} \big[ \log p(x\_i, z) \big] \bigg) + \mathbb{H}\big( q(z) \big) -\end{align*} -\$\$ - -and \$\$\mathbb{H} \big(q(z) \big)\$\$ denotes the [(differential) entropy](https://www.wikiwand.com/en/Differential_entropy) of \$\$q(z)\$\$. - -Assuming joint \$\$p(x\_i, z)\$\$ and the entropy \$\$\mathbb{H}\big(q(z)\big)\$\$ are both tractable, we can use a Monte-Carlo for the remaining expectation. This leaves us with the following tractable expression - -\$\$ -\underset{q}{\mathrm{argmin}} \ \mathrm{D\_{KL}} \big( q(z), p(z \mid \\{ x\_i \\}\_{i = 1}^n) \big) \approx \underset{q}{\mathrm{argmax}} \ \widehat{\mathrm{ELBO}}(q) -\$\$ - -where - -\$\$ -\widehat{\mathrm{ELBO}}(q) = \frac{1}{m} \bigg( \sum\_{k = 1}^m \sum\_{i = 1}^n \log p(x\_i, z\_k) \bigg) + \mathbb{H} \big(q(z)\big) \quad \text{where} \quad z\_k \sim q(z) \quad \forall k = 1, \dots, m -\$\$ - -Hence, as long as we can sample from \$\$q(z)\$\$ somewhat efficiently, we can indeed minimize the KL-divergence! Neat, eh? - -Sidenote: in the case where \$\$q(z)\$\$ is tractable but \$\$\mathbb{H} \big(q(z) \big)\$\$ is _not_ , we can use an Monte-Carlo estimate for this term too but this generally results in a higher-variance estimate. - -Also, I fooled you real good: the ELBO _isn't_ an arbitrary name, hah! In fact it's an abbreviation for the **expected lower bound (ELBO)** because it, uhmm, well, it's the _expected_ lower bound (remember \$\$\mathrm{D\_{KL}} \ge 0\$\$). Yup. - -## Maximizing the ELBO - -Finding the optimal \$\$q\$\$ over _all_ possible densities of course isn't feasible. Instead we consider a family of _parameterized_ densities \$\$\mathscr{D}\_{\Theta}\$\$ where \$\$\Theta\$\$ denotes the space of possible parameters. Each density in this family \$\$q\_{\theta} \in \mathscr{D}\_{\Theta}\$\$ is parameterized by a unique \$\$\theta \in \Theta\$\$. Moreover, we'll assume -1. \$\$q\_{\theta}(z)\$\$, i.e. evaluating the probability density \$\$q\$\$ at any point \$\$z\$\$, is differentiable -2. \$\$z \sim q\_{\theta}(z)\$\$, i.e. the process of sampling from \$\$q\_{\theta}(z)\$\$, is differentiable - -(1) is fairly straight-forward, but (2) is a bit tricky. What does it even mean for a _sampling process_ to be differentiable? This is quite an interesting problem in its own right and would require something like a [50-page paper to properly review the different approaches (highly recommended read)](https://arxiv.org/abs/1906.10652). - -We're going to make use of a particular such approach which goes under a bunch of different names: _reparametrization trick_, _path derivative_, etc. This refers to making the assumption that all elements \$\$q\_{\theta} \in \mathscr{Q}\_{\Theta}\$\$ can be considered as reparameterizations of some base density, say \$\$\bar{q}(z)\$\$. That is, if \$\$q\_{\theta} \in \mathscr{Q}\_{\Theta}\$\$ then - -\$\$ -z \sim q\_{\theta}(z) \quad \iff \quad z := g\_{\theta}(\tilde{z}) \quad \text{where} \quad \bar{z} \sim \bar{q}(z) -\$\$ - -for some function \$\$g\_{\theta}\$\$ differentiable wrt. \$\$\theta\$\$. So all \$\$q\_{\theta} \in \mathscr{Q}\_{\Theta}\$\$ are using the *same* reparameterization-function \$\$g\$\$ but each \$\$q\_{\theta}\$\$ correspond to different choices of \$\$\theta\$\$ for \$\$f\_{\theta}\$\$. - -Under this assumption we can differentiate the sampling process by taking the derivative of \$\$g\_{\theta}\$\$ wrt. \$\$\theta\$\$, and thus we can differentiate the entire \$\$\widehat{\mathrm{ELBO}}(q\_{\theta})\$\$ wrt. \$\$\theta\$\$! With the gradient available we can either try to solve for optimality either by setting the gradient equal to zero or maximize \$\$\widehat{\mathrm{ELBO}}(q\_{\theta})\$\$ stepwise by traversing \$\$\mathscr{Q}\_{\Theta}\$\$ in the direction of steepest ascent. For the sake of generality, we're going to go with the stepwise approach. - -With all this nailed down, we eventually reach the section on **Automatic Differentiation Variational Inference (ADVI)**. - -## Automatic Differentiation Variational Inference (ADVI) - -So let's revisit the assumptions we've made at this point: -1. The variational posterior \$\$q\_{\theta}\$\$ is in a parameterized family of densities denoted \$\$\mathscr{Q}\_{\Theta}\$\$, with \$\$\theta \in \Theta\$\$. -2. \$\$\mathscr{Q}\_{\Theta}\$\$ is a space of _reparameterizable_ densities with \$\$\bar{q}(z)\$\$ as the base-density. -3. The parameterization function \$\$g\_{\theta}\$\$ is differentiable wrt. \$\$\theta\$\$. -4. Evaluation of the probability density \$\$q\_{\theta}(z)\$\$ is differentiable wrt. \$\$\theta\$\$. -5. \$\$\mathbb{H}\big(q\_{\theta}(z)\big)\$\$ is tractable. -6. Evaluation of the joint density \$\$p(x, z)\$\$ is tractable and differentiable wrt. \$\$z\$\$ -7. The support of \$\$p(z \mid x)\$\$ is a subspace of the support of \$\$q(z)\$\$: \$\$\mathrm{supp}\big(p(z \mid x)\big) \subseteq \mathrm{supp}\big(q(z)\big)\$\$. - - -All of these are not *necessary* to do VI, but they are very convenient and results in a fairly flexible approach. One distribution which has a density satisfying all of the above assumptions _except_ (7) (we'll get back to this in second) for any tractable and differentiable \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ is the good ole' Gaussian/normal distribution: - -\$\$ -z \sim \mathcal{N}(\mu, \Sigma) \quad \iff \quad z = g\_{\mu, L}(\bar{z}) := \mu + L^T \tilde{z} \quad \text{where} \quad \bar{z} \sim \bar{q}(z) := \mathcal{N}(1\_d, I\_{d \times d}) -\$\$ - -where \$\$\Sigma = L L^T\$\$, with \$\$L\$\$ obtained from the Cholesky-decomposition. Abusing notation a bit, we're going to write - -\$\$ -\theta = (\mu, \Sigma) := (\mu\_1, \dots, \mu\_d, L\_{11}, \dots, L\_{1, d}, L\_{2, 1}, \dots, L\_{2, d}, \dots, L\_{d, 1}, \dots, L\_{d, d}) -\$\$ - -With this assumption we finally have a tractable expression for \$\$\widehat{\mathrm{ELBO}}(q\_{\mu, \Sigma})\$\$! Well, assuming (7) is holds. Since a Gaussian has non-zero probability on the entirety of \$\$\mathbb{R}^d\$\$, we also require \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$ to have non-zero probability on all of \$\$\mathbb{R}^d\$\$. - -Though not necessary, we'll often make a *mean-field* assumption for the variational posterior \$\$q(z)\$\$, i.e. assume independence between the latent variables. In this case, we'll write - -\$\$ -\theta = (\mu, \sigma^2) := (\mu\_1, \dots, \mu\_d, \sigma\_1^2, \dots, \sigma_d^2) -\$\$ - -### Examples -As a (trivial) example we could apply the approach described above to is the following generative model for \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$: - -\$\$ -\begin{align*} - m &\sim \mathcal{N}(0, 1) \\\\ - x\_i &\overset{\text{i.i.d.}}{=} \mathcal{N}(m, 1), \quad i = 1, \dots, n -\end{align*} -\$\$ - -In this case \$\$z = m\$\$ and we have the posterior defined \$\$p(m \mid \\{ x\_i \\}\_{i = 1}^n) = p(m) \prod\_{i = 1}^n p(x\_i \mid m)\$\$. Then the variational posterior would be - -\$\$ -q\_{\mu, \sigma} = \mathcal{N}(\mu, \sigma^2) \quad \text{where} \quad \mu \in \mathbb{R}, \ \sigma^2 \in \mathbb{R}^{ + } -\$\$ - -And since prior of \$\$m\$\$, \$\$\mathcal{N}(0, 1)\$\$, has non-zero probability on the entirety of \$\$\mathbb{R}\$\$, same as \$\$q(m)\$\$, i.e. assumption (7) above holds, everything is fine and life is good. - -But what about this generative model for \$\$p(z \mid \\{ x\_i \\}\_{i = 1}^n)\$\$: - -\$\$ -\begin{align*} - s &\sim \mathrm{InverseGamma}(2, 3) \\\\ - m &\sim \mathcal{N}(0, s) \\\\ - x\_i &\overset{\text{i.i.d.}}{=} \mathcal{N}(m, s), \quad i = 1, \dots, n -\end{align*} -\$\$ - -with posterior \$\$p(s, m \mid \\{ x\_i \\}\_{i = 1}^n) = p(s) p(m \mid s) \prod\_{i = 1}^n p(x\_i \mid s, m)\$\$ and the mean-field variational posterior \$\$q(s, m)\$\$ will be - -\$\$ -q\_{\mu\_1, \mu\_2, \sigma_1^2, \sigma_2^2}(s, m) = p\_{\mathcal{N}(\mu\_1, \sigma\_1^2)}(s) p\_{\mathcal{N}(\mu\_2, \sigma_2^2)}(m) -\$\$ - -where we've denoted the evaluation of the probability density of a Gaussian as \$\$p\_{\mathcal{N}(\mu, \sigma^2)}(x)\$\$. - -Observe that \$\$\mathrm{InverseGamma}(2, 3)\$\$ has non-zero probability only on \$\$\mathbb{R}^{ + } := (0, \infty)\$\$ which is clearly not all of \$\$\mathbb{R}\$\$ like \$\$q(s, m)\$\$ has, i.e. - -\$\$ -\mathrm{supp} \big( q(s, m) \big) \not\subseteq \mathrm{supp} \big( p(z \mid \\{ x\_i \\}\_{i = 1}^n) \big) -\$\$ - -Recall from the definition of the KL-divergence that when this is the case, the KL-divergence isn't well defined. This gets us to the *automatic* part of ADVI. - -### "Automatic"? How? - -For a lot of the standard (continuous) densities \$\$p\$\$ we can actually construct a probability density \$\$\tilde{p}\$\$ with non-zero probability on all of \$\$\mathbb{R}\$\$ by *transforming* the "constrained" probability density \$\$p\$\$ to \$\$\tilde{p}\$\$. In fact, in these cases this is a one-to-one relationship. As we'll see, this helps solve the support-issue we've been going on and on about. - -#### Transforming densities using change of variables - -If we want to compute the probability of \$\$x\$\$ taking a value in some set \$\$A \subseteq \mathrm{supp} \big( p(x) \big)\$\$, we have to integrate \$\$p(x)\$\$ over \$\$A\$\$, i.e. - -\$\$ -\mathbb{P}\_p(x \in A) = \int\_A p(x) \mathrm{d}x -\$\$ - -This means that if we have a differentiable bijection \$\$f: \mathrm{supp} \big( q(x) \big) \to \mathbb{R}^d\$\$ with differentiable inverse \$\$f^{-1}: \mathbb{R}^d \to \mathrm{supp} \big( p(x) \big)\$\$, we can perform a change of variables - -\$\$ -\mathbb{P}\_p(x \in A) = \int\_{f^{-1}(A)} p \big(f^{-1}(y) \big) \ \big| \det \mathcal{J}\_{f^{-1}}(y) \big| \ \mathrm{d}y -\$\$ - -where \$\$\mathcal{J}\_{f^{-1}}(x)\$\$ denotes the jacobian of \$\$f^{-1}\$\$ evaluted at \$\$x\$\$. Observe that this defines a probability distribution - -\$\$ -\mathbb{P}\_{\tilde{p}}\big(y \in f^{-1}(A) \big) = \int\_{f^{-1}(A)} \tilde{p}(y) \mathrm{d}y -\$\$ - -since \$\$f^{-1}\big(\mathrm{supp} (p(x)) \big) = \mathbb{R}^d\$\$ which has probability 1. This probability distribution has *density* \$\$\tilde{p}(y)\$\$ with \$\$\mathrm{supp} \big( \tilde{p}(y) \big) = \mathbb{R}^d\$\$, defined - -\$\$ -\tilde{p}(y) = p \big( f^{-1}(y) \big) \ \big| \det \mathcal{J}\_{f^{-1}}(y) \big| -\$\$ - -or equivalently - -\$\$ -\tilde{p} \big( f(x) \big) = \frac{p(x)}{\big| \det \mathcal{J}\_{f}(x) \big|} -\$\$ - -due to the fact that - -\$\$ -\big| \det \mathcal{J}\_{f^{-1}}(y) \big| = \big| \det \mathcal{J}\_{f}(x) \big|^{-1} -\$\$ - -*Note: it's also necessary that the log-abs-det-jacobian term is non-vanishing. This can for example be accomplished by assuming \$\$f\$\$ to also be elementwise monotonic.* - -#### Back to VI - -So why is this is useful? Well, we're looking to generalize our approach using a normal distribution to cases where the supports don't match up. How about defining \$\$q(z)\$\$ by - -\$\$ -\begin{align*} - \eta &\sim \mathcal{N}(\mu, \Sigma) \\\\ - z &= f^{-1}(\eta) -\end{align*} -\$\$ - -where \$\$f^{-1}: \mathbb{R}^d \to \mathrm{supp} \big( p(z \mid x) \big)\$\$ is a differentiable bijection with differentiable inverse. Then \$\$z \sim q\_{\mu, \Sigma}(z) \implies z \in \mathrm{supp} \big( p(z \mid x) \big)\$\$ as we wanted. The resulting variational density is - -\$\$ -q\_{\mu, \Sigma}(z) = p\_{\mathcal{N}(\mu, \Sigma)}\big( f(z) \big) \ \big| \det \mathcal{J}\_{f}(z) \big| -\$\$ - -Note that the way we've constructed \$\$q(z)\$\$ here is basically a reverse of the approach we described above. Here we sample from a distribution with support on \$\$\mathbb{R}\$\$ and transform *to* \$\$\mathrm{supp} \big( p(z \mid x) \big)\$\$. - -If we want to write the ELBO explicitly in terms of \$\$\eta\$\$ rather than \$\$z\$\$, the first term in the ELBO becomes - -\$\$ -\begin{align*} - \mathbb{E}\_{z \sim q\_{\mu, \Sigma}(z)} \big[ \log p(x\_i, z) \big] &= \mathbb{E}\_{\eta \sim \mathcal{N}(\mu, \Sigma)} \Bigg[ \log \frac{p\big(x\_i, f^{-1}(\eta) \big)}{\big| \det \mathcal{J}\_{f^{-1}}(\eta) \big|} \Bigg] \\\\ - &= \mathbb{E}\_{\eta \sim \mathcal{N}(\mu, \Sigma)} \big[ \log p\big(x\_i, f^{-1}(\eta) \big) \big] - \mathbb{E}\_{\eta \sim \mathcal{N}(\mu, \Sigma)} \big[ \big| \det \mathcal{J}\_{f^{-1}}(\eta) \big| \big] -\end{align*} -\$\$ - -The entropy is invariant under change of variables, thus \$\$\mathbb{H} \big(q\_{\mu, \Sigma}(z)\big)\$\$ is simply the entropy of the normal distribution which is known analytically. - -Hence, the resulting empirical estimate of the ELBO is - -\$\$ -\begin{align*} -\widehat{\mathrm{ELBO}}(q\_{\mu, \Sigma}) &= \frac{1}{m} \bigg( \sum\_{k = 1}^m \sum\_{i = 1}^n \Big(\log p\big(x\_i, f^{-1}(\eta\_k)\big) - \log \big| \det \mathcal{J}\_{f^{-1}}(\eta\_k) \big| \Big) \bigg) + \mathbb{H} \big(p\_{\mathcal{N}(\mu, \Sigma)}(z)\big) \\\\ -& \text{where} \quad z\_k \sim \mathcal{N}(\mu, \Sigma) \quad \forall k = 1, \dots, m -\end{align*} -\$\$ - -And maximizing this wrt. \$\$\mu\$\$ and \$\$\Sigma\$\$ is what's referred to as **Automatic Differentation Variational Inference (ADVI)**! - -Now if you want to try it out, [check out the tutorial on how to use ADVI in Turing.jl](../../tutorials/9-variationalinference)! diff --git a/src/library/advancedhmc.md b/src/library/advancedhmc.md deleted file mode 100644 index 5742e4cff..000000000 --- a/src/library/advancedhmc.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: AdvancedHMC -permalink: /docs/library/advancedhmc/ -toc: true ---- - -## Index - -```@index -Modules = [AdvancedHMC] -``` - -## Functions - -```@autodocs -Modules = [AdvancedHMC] -Order = [:function] -``` - -## Types - -```@autodocs -Modules = [AdvancedHMC] -Order = [:type] -``` \ No newline at end of file diff --git a/src/library/api.md b/src/library/api.md deleted file mode 100644 index 1d3d80dcb..000000000 --- a/src/library/api.md +++ /dev/null @@ -1,56 +0,0 @@ ---- -title: API -permalink: /docs/library/ -toc: true ---- - -```@meta -CurrentModule = Turing -``` -## Index - -```@index -Modules = [Turing, Turing.Core, Turing.Inference, Libtask] -``` - -## Modelling - -```@docs -@model -``` - -## Samplers - -```@docs -Sampler -Gibbs -HMC -HMCDA -IS -MH -NUTS -PG -SMC -``` - -## Distributions - -```@docs -Flat -FlatPos -BinomialLogit -VecBinomialLogit -OrderedLogistic -``` - - -## Data Structures -```@docs -TArray -``` - -## Utilities -```@docs -tzeros -``` - diff --git a/src/library/bijectors.md b/src/library/bijectors.md deleted file mode 100644 index 9fb4eaecf..000000000 --- a/src/library/bijectors.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: Bijectors -permalink: /docs/library/bijectors/ -toc: true ---- - -## Index - -```@index -Modules = [Bijectors] -``` - -## Functions - -```@autodocs -Modules = [Bijectors] -Order = [:function] -``` - -## Types - -```@autodocs -Modules = [Bijectors] -Order = [:type] -``` \ No newline at end of file diff --git a/src/tutorials/index.md b/src/tutorials/index.md deleted file mode 100644 index a37d31ad7..000000000 --- a/src/tutorials/index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: Tutorials ---- - -# Tutorials - -This section contains tutorials on how to implement common models in Turing. If you prefer to have an interactive Jupyter notebook, please fork or download the [TuringTutorials](https://github.com/TuringLang/TuringTutorials) repository. - - -A list of all the tutorials available can be found to the left. The [introduction]({{site.baseurl}}/tutorials/0-introduction) tutorial contains an introduction to coin flipping with Turing and a brief overview of probabalistic programming. - - -Tutorials are under continuous development, but there are some older version available at the TuringTutorials within the [`old-notebooks`](https://github.com/TuringLang/TuringTutorials/tree/master/old-notebooks/) section. Some of these were built using prior versions of Turing and may not function correctly, but they can assist in the syntax used for common models. - - -If there is a tutorial you would like to request, please open an issue on the [TuringTutorials](https://github.com/TuringLang/TuringTutorials) repository. - diff --git a/src/using-turing/advanced.md b/src/using-turing/advanced.md deleted file mode 100644 index 398a1e0f3..000000000 --- a/src/using-turing/advanced.md +++ /dev/null @@ -1,196 +0,0 @@ ---- -title: Advanced Usage ---- - -# Advanced Usage - -## How to Define a Customized Distribution - - -Turing.jl supports the use of distributions from the Distributions.jl package. By extension it also supports the use of customized distributions, by defining them as subtypes of `Distribution` type of the Distributions.jl package, as well as corresponding functions. - - -Below shows a workflow of how to define a customized distribution, using our own implementation of a simple `Uniform` distribution as a simple example. - - -### 1. Define the Distribution Type - - -First, define a type of the distribution, as a subtype of a corresponding distribution type in the Distributions.jl package. - - -```julia -struct CustomUniform <: ContinuousUnivariateDistribution -end -``` - -### 2. Implement Sampling and Evaluation of the log-pdf - - -Second, define `rand` and `logpdf`, which will be used to run the model. - - -```julia -Distributions.rand(rng::AbstractRNG, d::CustomUniform) = rand(rng) # sample in [0, 1] -Distributions.logpdf(d::CustomUniform, x::Real) = zero(x) # p(x) = 1 → logp(x) = 0 -``` - -### 3. Define Helper Functions - - -In most cases, it may be required to define some helper functions. - -#### 3.1 Domain Transformation - -Certain samplers, such as `HMC`, require the domain of the priors to be unbounded. Therefore, to use our `CustomUniform` as a prior in a model we also need to define how to transform samples from `[0, 1]` to `ℝ`. To do this, we simply need to define the corresponding `Bijector` from `Bijectors.jl`, which is what `Turing.jl` uses internally to deal with constrained distributions. - -To transform from `[0, 1]` to `ℝ` we can use the `Logit` bijector: - -```julia -Bijectors.bijector(d::CustomUniform) = Logit(0., 1.) -``` - -You'd do the exact same thing for `ContinuousMultivariateDistribution` and `ContinuousMatrixDistribution`. For example, `Wishart` defines a distribution over positive-definite matrices and so `bijector` returns a `PDBijector` when called with a `Wishart` distribution as an argument. For discrete distributions, there is no need to define a bijector; the `Identity` bijector is used by default. - -Alternatively, for `UnivariateDistribution` we can define the `minimum` and `maximum` of the distribution - -```julia -Distributions.minimum(d::CustomUniform) = 0. -Distributions.maximum(d::CustomUniform) = 1. -``` - -and `Bijectors.jl` will return a default `Bijector` called `TruncatedBijector` which makes use of `minimum` and `maximum` derive the correct transformation. - -Internally, Turing basically does the following when it needs to convert a constrained distribution to an unconstrained distribution, e.g. when sampling using `HMC`: -```julia -b = bijector(dist) -transformed_dist = transformed(dist, b) # results in distribution with transformed support + correction for logpdf -``` -and then we can call `rand` and `logpdf` as usual, where -- `rand(transformed_dist)` returns a sample in the unconstrained space, and -- `logpdf(transformed_dist, y)` returns the log density of the original distribution, but with `y` living in the unconstrained space. - -To read more about Bijectors.jl, check out [the project README](https://github.com/TuringLang/Bijectors.jl). - -#### 3.2 Vectorization Support - - -The vectorization syntax follows `rv ~ [distribution]`, which requires `rand` and `logpdf` to be called on multiple data points at once. An appropriate implementation for `Flat` is shown below. - - -```julia -Distributions.logpdf(d::Flat, x::AbstractVector{<:Real}) = zero(x) -``` - -## Update the accumulated log probability in the model definition - -Turing accumulates log probabilities internally in an internal data structure that is accessible through -the internal variable `_varinfo` inside of the model definition (see below for more details about model internals). -However, since users should not have to deal with internal data structures, a macro `Turing.@addlogprob!` is provided -that increases the accumulated log probability. For instance, this allows you to -[include arbitrary terms in the likelihood](https://github.com/TuringLang/Turing.jl/issues/1332) - -```julia -using Turing - -myloglikelihood(x, μ) = loglikelihood(Normal(μ, 1), x) - -@model function demo(x) - μ ~ Normal() - Turing.@addlogprob! myloglikelihood(x, μ) -end -``` - -and to [reject samples](https://github.com/TuringLang/Turing.jl/issues/1328): - -```julia -using Turing -using LinearAlgebra - -@model function demo(x) - m ~ MvNormal(length(x)) - if dot(m, x) < 0 - Turing.@addlogprob! -Inf - # Exit the model evaluation early - return - end - - x ~ MvNormal(m, 1.0) - return -end -``` - -Note that `@addlogprob!` always increases the accumulated log probability, regardless of the provided -sampling context. For instance, if you do not want to apply `Turing.@addlogprob!` when evaluating the -prior of your model but only when computing the log likelihood and the log joint probability, then you -should [check the type of the internal variable `_context`](https://github.com/TuringLang/DynamicPPL.jl/issues/154) -such as - -```julia -if !isa(_context, Turing.PriorContext) - Turing.@addlogprob! myloglikelihood(x, μ) -end -``` - -## Model Internals - - -The `@model` macro accepts a function definition and rewrites it such that call of the function generates a `Model` struct for use by the sampler. Models can be constructed by hand without the use of a macro. Taking the `gdemo` model as an example, the macro-based definition - -```julia -using Turing - -@model function gdemo(x) - # Set priors. - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - - # Observe each value of x. - @. x ~ Normal(m, sqrt(s)) -end - -model = gdemo([1.5, 2.0]) -``` - -is equivalent to the macro-free version - -```julia -using Turing - -# Create the model function. -function modelf(rng, model, varinfo, sampler, context, x) - # Assume s has an InverseGamma distribution. - s = Turing.DynamicPPL.tilde_assume( - rng, - context, - sampler, - InverseGamma(2, 3), - Turing.@varname(s), - (), - varinfo, - ) - - # Assume m has a Normal distribution. - m = Turing.DynamicPPL.tilde_assume( - rng, - context, - sampler, - Normal(0, sqrt(s)), - Turing.@varname(m), - (), - varinfo, - ) - - # Observe each value of x[i] according to a Normal distribution. - Turing.DynamicPPL.dot_tilde_observe(context, sampler, Normal(m, sqrt(s)), x, varinfo) -end - -# Instantiate a Model object with our data variables. -model = Turing.Model(modelf, (x = [1.5, 2.0],)) -``` - -## Task Copying - - -Turing [copies](https://github.com/JuliaLang/julia/issues/4085) Julia tasks to deliver efficient inference algorithms, but it also provides alternative slower implementation as a fallback. Task copying is enabled by default. Task copying requires us to use the `CTask` facility which is provided by [Libtask](https://github.com/TuringLang/Libtask.jl) to create tasks. - diff --git a/src/using-turing/autodiff.md b/src/using-turing/autodiff.md deleted file mode 100644 index 9573b4b32..000000000 --- a/src/using-turing/autodiff.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: Automatic Differentiation - ---- - -# Automatic Differentiation - -## Switching AD Modes - - -Turing supports four packages of automatic differentiation (AD) in the back end during sampling. The default AD backend is [ForwardDiff](https://github.com/JuliaDiff/ForwardDiff.jl) for forward-mode AD. Three reverse-mode AD backends are also supported, namely [Tracker](https://github.com/FluxML/Tracker.jl), [Zygote](https://github.com/FluxML/Zygote.jl) and [ReverseDiff](https://github.com/JuliaDiff/ReverseDiff.jl). `Zygote` and `ReverseDiff` are supported optionally if explicitly loaded by the user with `using Zygote` or `using ReverseDiff` next to `using Turing`. - -To switch between the different AD backends, one can call function `Turing.setadbackend(backend_sym)`, where `backend_sym` can be `:forwarddiff` (`ForwardDiff`), `:tracker` (`Tracker`), `:zygote` (`Zygote`) or `:reversediff` (`ReverseDiff.jl`). When using `ReverseDiff`, to compile the tape only once and cache it for later use, the user needs to load [Memoization.jl](https://github.com/marius311/Memoization.jl) first with `using Memoization` then call `Turing.setrdcache(true)`. However, note that the use of caching in certain types of models can lead to incorrect results and/or errors. Models for which the compiled tape can be safely cached are models with fixed size loops and no run-time if statements. Compile-time if statements are fine. To empty the cache, you can call `Turing.emptyrdcache()`. - - -## Compositional Sampling with Differing AD Modes - - -Turing supports intermixed automatic differentiation methods for different variable spaces. The snippet below shows using `ForwardDiff` to sample the mean (`m`) parameter, and using the Tracker-based `TrackerAD` autodiff for the variance (`s`) parameter: - - -```julia -using Turing - -# Define a simple Normal model with unknown mean and variance. -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Sample using Gibbs and varying autodiff backends. -c = sample( - gdemo(1.5, 2), - Gibbs( - HMC{Turing.ForwardDiffAD{1}}(0.1, 5, :m), - HMC{Turing.TrackerAD}(0.1, 5, :s) - ), - 1000, -) -``` - - -Generally, `TrackerAD` is faster when sampling from variables of high dimensionality (greater than 20) and `ForwardDiffAD` is more efficient for lower-dimension variables. This functionality allows those who are performance sensistive to fine tune their automatic differentiation for their specific models. - - -If the differentation method is not specified in this way, Turing will default to using whatever the global AD backend is. Currently, this defaults to `ForwardDiff`. - diff --git a/src/using-turing/dynamichmc.md b/src/using-turing/dynamichmc.md deleted file mode 100644 index eb226be94..000000000 --- a/src/using-turing/dynamichmc.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -title: Using DynamicHMC ---- - -# Using DynamicHMC - -Turing supports the use of [DynamicHMC](https://github.com/tpapp/DynamicHMC.jl) as a sampler through the `DynamicNUTS` function. - - -`DynamicNUTS` is not appropriate for use in compositional inference. If you intend to use [Gibbs]({{site.baseurl}}/docs/library/#Turing.Inference.Gibbs) sampling, you must use Turing's native `NUTS` function. - - -To use the `DynamicNUTS` function, you must import the `DynamicHMC` package as well as Turing. Turing does not formally require `DynamicHMC` but will include additional functionality if both packages are present. - -Here is a brief example of how to apply `DynamicNUTS`: - - -```julia -# Import Turing and DynamicHMC. -using LogDensityProblems, DynamicHMC, Turing - -# Model definition. -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Pull 2,000 samples using DynamicNUTS. -chn = sample(gdemo(1.5, 2.0), DynamicNUTS(), 2000) -``` diff --git a/src/using-turing/gen-sampler-viz.jl b/src/using-turing/gen-sampler-viz.jl deleted file mode 100644 index 023c7cd98..000000000 --- a/src/using-turing/gen-sampler-viz.jl +++ /dev/null @@ -1,100 +0,0 @@ -ENV["GKS_ENCODING"] = "utf-8" # Allows the use of unicode characters in Plots.jl -using Plots -using StatsPlots -using Turing -using Bijectors -using Random -using DynamicPPL: getlogp, settrans!, getval, reconstruct, vectorize, setval! - -# Set a seed. -Random.seed!(0) - -# Define a strange model. -@model gdemo(x) = begin - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - bumps = sin(m) + cos(m) - m = m + 5*bumps - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end - return s, m -end - -# Define our data points. -x = [1.5, 2.0, 13.0, 2.1, 0.0] - -# Set up the model call, sample from the prior. -model = gdemo(x) -vi = Turing.VarInfo(model) - -# Convert the variance parameter to the real line before sampling. -# Note: We only have to do this here because we are being very hands-on. -# Turing will handle all of this for you during normal sampling. -dist = InverseGamma(2,3) -svn = vi.metadata.s.vns[1] -mvn = vi.metadata.m.vns[1] -setval!(vi, vectorize(dist, Bijectors.link(dist, reconstruct(dist, getval(vi, svn)))), svn) -settrans!(vi, true, svn) - -# Evaluate surface at coordinates. -function evaluate(m1, m2) - spl = Turing.SampleFromPrior() - vi[svn] = [m1] - vi[mvn] = [m2] - model(vi, spl) - getlogp(vi) -end - -function plot_sampler(chain; label="") - # Extract values from chain. - val = get(chain, [:s, :m, :lp]) - ss = link.(Ref(InverseGamma(2, 3)), val.s) - ms = val.m - lps = val.lp - - # How many surface points to sample. - granularity = 100 - - # Range start/stop points. - spread = 0.5 - σ_start = minimum(ss) - spread * std(ss); σ_stop = maximum(ss) + spread * std(ss); - μ_start = minimum(ms) - spread * std(ms); μ_stop = maximum(ms) + spread * std(ms); - σ_rng = collect(range(σ_start, stop=σ_stop, length=granularity)) - μ_rng = collect(range(μ_start, stop=μ_stop, length=granularity)) - - # Make surface plot. - p = surface(σ_rng, μ_rng, evaluate, - camera=(30, 65), - # ticks=nothing, - colorbar=false, - color=:inferno, - title=label) - - line_range = 1:length(ms) - - scatter3d!(ss[line_range], ms[line_range], lps[line_range], - mc =:viridis, marker_z=collect(line_range), msw=0, - legend=false, colorbar=false, alpha=0.5, - xlabel="σ", ylabel="μ", zlabel="Log probability", - title=label) - - return p -end; - -samplers = [ - (Gibbs(HMC(0.01, 5, :s), PG(20, :m)), "Gibbs{HMC, PG}"), - (HMC(0.01, 10), "HMC"), - (HMCDA(200, 0.65, 0.3), "HMCDA"), - (MH(), "MH()"), - (NUTS(0.65), "NUTS(0.65)"), - (NUTS(0.95), "NUTS(0.95)"), - (NUTS(0.2), "NUTS(0.2)"), - (PG(20), "PG(20)"), - (PG(50), "PG(50)")] - -for (i, (spl, spl_name)) in enumerate(samplers) - c = sample(model, spl, 1000) - p = plot_sampler(c, label="$spl_name") - savefig(joinpath(@__DIR__, "sampler-figs", "samplers-$i.svg")) -end diff --git a/src/using-turing/get-started.md b/src/using-turing/get-started.md deleted file mode 100644 index f0bf1acba..000000000 --- a/src/using-turing/get-started.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -title: Getting Started ---- - -# Getting Started - - -## Installation - -To use Turing, you need to install Julia first and then install Turing. - - -### Install Julia - -You will need to install Julia 1.3 or greater, which you can get from [the official Julia website](http://julialang.org/downloads/). - - -### Install Turing.jl - -Turing is an officially registered Julia package, so you can install a stable version of Turing by running the following in the Julia REPL: - -```julia -julia> ] add Turing -``` - -You can check if all tests pass by running - -```julia -julia> ] test Turing -``` - -### Example - -Here's a simple example showing the package in action: - - -```julia -using Turing -using StatsPlots - -# Define a simple Normal model with unknown mean and variance. -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end - -# Run sampler, collect results -chn = sample(gdemo(1.5, 2), HMC(0.1, 5), 1000) - -# Summarise results -describe(chn) - -# Plot and save results -p = plot(chn) -savefig("gdemo-plot.png") -``` diff --git a/src/using-turing/guide.md b/src/using-turing/guide.md deleted file mode 100644 index a25c3469e..000000000 --- a/src/using-turing/guide.md +++ /dev/null @@ -1,640 +0,0 @@ ---- -title: Guide ---- - -# Guide - - -## Basics - - -### Introduction - -A probabilistic program is Julia code wrapped in a `@model` macro. It can use arbitrary Julia code, but to ensure correctness of inference it should not have external effects or modify global state. Stack-allocated variables are safe, but mutable heap-allocated objects may lead to subtle bugs when using task copying. To help avoid those we provide a Turing-safe datatype `TArray` that can be used to create mutable arrays in Turing programs. - - -To specify distributions of random variables, Turing programs should use the `~` notation: - - -`x ~ distr` where `x` is a symbol and `distr` is a distribution. If `x` is undefined in the model function, inside the probabilistic program, this puts a random variable named `x`, distributed according to `distr`, in the current scope. `distr` can be a value of any type that implements `rand(distr)`, which samples a value from the distribution `distr`. If `x` is defined, this is used for conditioning in a style similar to [Anglican](https://probprog.github.io/anglican/index.html) (another PPL). In this case, `x` is an observed value, assumed to have been drawn from the distribution `distr`. The likelihood is computed using `logpdf(distr,y)`. The observe statements should be arranged so that every possible run traverses all of them in exactly the same order. This is equivalent to demanding that they are not placed inside stochastic control flow. - - -Available inference methods include Importance Sampling (IS), Sequential Monte Carlo (SMC), Particle Gibbs (PG), Hamiltonian Monte Carlo (HMC), Hamiltonian Monte Carlo with Dual Averaging (HMCDA) and The No-U-Turn Sampler (NUTS). - - -### Simple Gaussian Demo - -Below is a simple Gaussian demo illustrate the basic usage of Turing.jl. - - -```julia -# Import packages. -using Turing -using StatsPlots - -# Define a simple Normal model with unknown mean and variance. -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end -``` - - -Note: As a sanity check, the expectation of `s` is 49/24 (2.04166666...) and the expectation of `m` is 7/6 (1.16666666...). - - -We can perform inference by using the `sample` function, the first argument of which is our probabalistic program and the second of which is a sampler. More information on each sampler is located in the [API]({{site.baseurl}}/docs/library). - - -```julia -# Run sampler, collect results. -c1 = sample(gdemo(1.5, 2), SMC(), 1000) -c2 = sample(gdemo(1.5, 2), PG(10), 1000) -c3 = sample(gdemo(1.5, 2), HMC(0.1, 5), 1000) -c4 = sample(gdemo(1.5, 2), Gibbs(PG(10, :m), HMC(0.1, 5, :s)), 1000) -c5 = sample(gdemo(1.5, 2), HMCDA(0.15, 0.65), 1000) -c6 = sample(gdemo(1.5, 2), NUTS(0.65), 1000) -``` - - -The `MCMCChains` module (which is re-exported by Turing) provides plotting tools for the `Chain` objects returned by a `sample` function. See the [MCMCChains](https://github.com/TuringLang/MCMCChains.jl) repository for more information on the suite of tools available for diagnosing MCMC chains. - - -```julia -# Summarise results -describe(c3) - -# Plot results -plot(c3) -savefig("gdemo-plot.png") -``` - - -The arguments for each sampler are: - - - * SMC: number of particles. - * PG: number of particles, number of iterations. - * HMC: leapfrog step size, leapfrog step numbers. - * Gibbs: component sampler 1, component sampler 2, ... - * HMCDA: total leapfrog length, target accept ratio. - * NUTS: number of adaptation steps (optional), target accept ratio. - - -For detailed information on the samplers, please review Turing.jl's [API]({{site.baseurl}}/docs/library) documentation. - - -### Modelling Syntax Explained - - -Using this syntax, a probabilistic model is defined in Turing. The model function generated by Turing can then be used to condition the model onto data. Subsequently, the sample function can be used to generate samples from the posterior distribution. - - -In the following example, the defined model is conditioned to the date (arg*1 = 1, arg*2 = 2) by passing (1, 2) to the model function. - - -```julia -@model function model_name(arg_1, arg_2) - ... -end -``` - - -The conditioned model can then be passed onto the sample function to run posterior inference. - - -```julia -model_func = model_name(1, 2) -chn = sample(model_func, HMC(..)) # Perform inference by sampling using HMC. -``` - - -The returned chain contains samples of the variables in the model. - - -```julia -var_1 = mean(chn[:var_1]) # Taking the mean of a variable named var_1. -``` - - -The key (`:var_1`) can be a `Symbol` or a `String`. For example, to fetch `x[1]`, one can use `chn[Symbol("x[1]")` or `chn["x[1]"]`. -If you want to retrieve all parameters associated with a specific symbol, you can use `group`. As an example, if you have the -parameters `"x[1]"`, `"x[2]"`, and `"x[3]"`, calling `group(chn, :x)` or `group(chn, "x")` will return a new chain with only `"x[1]"`, `"x[2]"`, and `"x[3]"`. - - -Turing does not have a declarative form. More generally, the order in which you place the lines of a `@model` macro matters. For example, the following example works: - - -```julia -# Define a simple Normal model with unknown mean and variance. -@model function model_function(y) - s ~ Poisson(1) - y ~ Normal(s, 1) - return y -end - -sample(model_function(10), SMC(), 100) -``` - - -But if we switch the `s ~ Poisson(1)` and `y ~ Normal(s, 1)` lines, the model will no longer sample correctly: - - -```julia -# Define a simple Normal model with unknown mean and variance. -@model function model_function(y) - y ~ Normal(s, 1) - s ~ Poisson(1) - return y -end - -sample(model_function(10), SMC(), 100) -``` - - - -### Sampling Multiple Chains - -Turing supports distributed and threaded parallel sampling. To do so, call `sample(model, sampler, parallel_type, n, n_chains)`, where `parallel_type` can be either `MCMCThreads()` or `MCMCDistributed()` for thread and parallel sampling, respectively. - -Having multiple chains in the same object is valuable for evaluating convergence. Some diagnostic functions like `gelmandiag` require multiple chains. - -If you do not want parallelism or are on an older version Julia, you can sample multiple chains with the `mapreduce` function: - -```julia -# Replace num_chains below with however many chains you wish to sample. -chains = mapreduce(c -> sample(model_fun, sampler, 1000), chainscat, 1:num_chains) -``` - -The `chains` variable now contains a `Chains` object which can be indexed by chain. To pull out the first chain from the `chains` object, use `chains[:,:,1]`. The method is the same if you use either of the below parallel sampling methods. - -#### Multithreaded sampling - -If you wish to perform multithreaded sampling and are running Julia 1.3 or greater, you can call `sample` with the following signature: - -```julia -using Turing - -@model function gdemo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end - -model = gdemo([1.5, 2.0]) - -# Sample four chains using multiple threads, each with 1000 samples. -sample(model, NUTS(), MCMCThreads(), 1000, 4) -``` - -Be aware that Turing cannot add threads for you -- you must have started your Julia instance with multiple threads to experience any kind of parallelism. See the [Julia documentation](https://docs.julialang.org/en/v1/manual/parallel-computing/#man-multithreading-1) for details on how to achieve this. - -#### Distributed sampling - -To perform distributed sampling (using multiple processes), you must first import `Distributed`. - -Process parallel sampling can be done like so: - -```julia -# Load Distributed to add processes and the @everywhere macro. -using Distributed - -# Load Turing. -using Turing - -# Add four processes to use for sampling. -addprocs(4) - -# Initialize everything on all the processes. -# Note: Make sure to do this after you've already loaded Turing, -# so each process does not have to precompile. -# Parallel sampling may fail silently if you do not do this. -@everywhere using Turing - -# Define a model on all processes. -@everywhere @model function gdemo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end - -# Declare the model instance everywhere. -@everywhere model = gdemo([1.5, 2.0]) - -# Sample four chains using multiple processes, each with 1000 samples. -sample(model, NUTS(), MCMCDistributed(), 1000, 4) -``` - -### Sampling from an Unconditional Distribution (The Prior) - -Turing allows you to sample from a declared model's prior. If you wish to draw a chain from the prior to inspect your prior distributions, you can simply run - -```julia -chain = sample(model, Prior(), n_samples) -``` - -You can also run your model (as if it were a function) from the prior distribution, by calling the model without specifying inputs or a sampler. In the below example, we specify a `gdemo` model which returns two variables, `x` and `y`. The model includes `x` and `y` as arguments, but calling the function without passing in `x` or `y` means that Turing's compiler will assume they are missing values to draw from the relevant distribution. The `return` statement is necessary to retrieve the sampled `x` and `y` values. - - -```julia -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) - return x, y -end -``` - - -Assign the function with `missing` inputs to a variable, and Turing will produce a sample from the prior distribution. - - -```julia -# Samples from p(x,y) -g_prior_sample = gdemo(missing, missing) -g_prior_sample() -``` - - -Output: - - -``` -(0.685690547873451, -1.1972706455914328) -``` - - -### Sampling from a Conditional Distribution (The Posterior) - - -#### Treating observations as random variables - - -Inputs to the model that have a value `missing` are treated as parameters, aka random variables, to be estimated/sampled. This can be useful if you want to simulate draws for that parameter, or if you are sampling from a conditional distribution. Turing supports the following syntax: - - -```julia -@model function gdemo(x, ::Type{T} = Float64) where {T} - if x === missing - # Initialize `x` if missing - x = Vector{T}(undef, 2) - end - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end - -# Construct a model with x = missing -model = gdemo(missing) -c = sample(model, HMC(0.01, 5), 500) -``` - -Note the need to initialize `x` when missing since we are iterating over its elements later in the model. The generated values for `x` can be extracted from the `Chains` object using `c[:x]`. - - -Turing also supports mixed `missing` and non-`missing` values in `x`, where the missing ones will be treated as random variables to be sampled while the others get treated as observations. For example: - - -```julia -@model function gdemo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end - -# x[1] is a parameter, but x[2] is an observation -model = gdemo([missing, 2.4]) -c = sample(model, HMC(0.01, 5), 500) -``` - - -#### Default Values - - -Arguments to Turing models can have default values much like how default values work in normal Julia functions. For instance, the following will assign `missing` to `x` and treat it as a random variable. If the default value is not `missing`, `x` will be assigned that value and will be treated as an observation instead. - -```julia -using Turing - -@model function generative(x = missing, ::Type{T} = Float64) where {T <: Real} - if x === missing - # Initialize x when missing - x = Vector{T}(undef, 10) - end - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - for i in 1:length(x) - x[i] ~ Normal(m, sqrt(s)) - end - return s, m -end - -m = generative() -chain = sample(m, HMC(0.01, 5), 1000) -``` - - -#### Access Values inside Chain - - -You can access the values inside a chain several ways: - -1. Turn them into a `DataFrame` object -2. Use their raw `AxisArray` form -3. Create a three-dimensional `Array` object - -For example, let `c` be a `Chain`: - 1. `DataFrame(c)` converts `c` to a `DataFrame`, - 2. `c.value` retrieves the values inside `c` as an `AxisArray`, and - 3. `c.value.data` retrieves the values inside `c` as a 3D `Array`. - - -#### Variable Types and Type Parameters - - -The element type of a vector (or matrix) of random variables should match the `eltype` of the its prior distribution, `<: Integer` for discrete distributions and `<: AbstractFloat` for continuous distributions. Moreover, if the continuous random variable is to be sampled using a Hamiltonian sampler, the vector's element type needs to either be: - 1. `Real` to enable auto-differentiation through the model which uses special number types that are sub-types of `Real`, or - 2. Some type parameter `T` defined in the model header using the type parameter syntax, e.g. `gdemo(x, ::Type{T} = Float64) where {T} = begin`. -Similarly, when using a particle sampler, the Julia variable used should either be: - 1. A `TArray`, or - 2. An instance of some type parameter `T` defined in the model header using the type parameter syntax, e.g. `gdemo(x, ::Type{T} = Vector{Float64}) where {T} = begin`. - - -### Querying Probabilities from Model or Chain - - -Consider the following `gdemo` model: -```julia -@model function gdemo(x, y) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - x ~ Normal(m, sqrt(s)) - y ~ Normal(m, sqrt(s)) -end -``` - -The following are examples of valid queries of the `Turing` model or chain: - -- `prob"x = 1.0, y = 1.0 | model = gdemo, s = 1.0, m = 1.0"` calculates the likelihood of `x = 1` and `y = 1` given `s = 1` and `m = 1`. - -- `prob"s = 1.0, m = 1.0 | model = gdemo, x = nothing, y = nothing"` calculates the joint probability of `s = 1` and `m = 1` ignoring `x` and `y`. `x` and `y` are ignored so they can be optionally dropped from the RHS of `|`, but it is recommended to define them. - -- `prob"s = 1.0, m = 1.0, x = 1.0 | model = gdemo, y = nothing"` calculates the joint probability of `s = 1`, `m = 1` and `x = 1` ignoring `y`. - -- `prob"s = 1.0, m = 1.0, x = 1.0, y = 1.0 | model = gdemo"` calculates the joint probability of all the variables. - -- After the MCMC sampling, given a `chain`, `prob"x = 1.0, y = 1.0 | chain = chain, model = gdemo"` calculates the element-wise likelihood of `x = 1.0` and `y = 1.0` for each sample in `chain`. - -- If `save_state=true` was used during sampling (i.e., `sample(model, sampler, N; save_state=true)`), you can simply do `prob"x = 1.0, y = 1.0 | chain = chain"`. - -In all the above cases, `logprob` can be used instead of `prob` to calculate the log probabilities instead. - - -### Maximum likelihood and maximum a posterior estimates - -Turing provides support for two mode estimation techniques, [maximum likelihood estimation](https://en.wikipedia.org/wiki/Maximum_likelihood_estimation) (MLE) and [maximum a posterior](https://en.wikipedia.org/wiki/Maximum_a_posteriori_estimation) (MAP) estimation. Optimization is performed by the [Optim.jl](https://github.com/JuliaNLSolvers/Optim.jl) package. Mode estimation is currently a optional tool, and will not be available to you unless you have manually installed Optim and loaded the package with a `using` statement. To install Optim, run `import Pkg; Pkg.add("Optim")`. - -Mode estimation only works when all model parameters are continuous -- discrete parameters cannot be estimated with MLE/MAP as of yet. - -To understand how mode estimation works, let us first load Turing and Optim to enable mode estimation, and then declare a model: - -```julia -# Note that loading Optim explicitly is required for mode estimation to function, -# as Turing does not load the opimization suite unless Optim is loaded as well. -using Turing -using Optim - -@model function gdemo(x) - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end -end -``` - -Once the model is defined, we can construct a model instance as we normally would: - -```julia -# Create some data to pass to the model. -data = [1.5, 2.0] - -# Instantiate the gdemo model with our data. -model = gdemo(data) -``` - -Mode estimation is typically quick and easy at this point. Turing extends the function `Optim.optimize` and accepts the structs `MLE()` or `MAP()`, which inform Turing whether to provide an MLE or MAP estimate, respectively. By default, the [LBFGS optimizer](https://julianlsolvers.github.io/Optim.jl/stable/#algo/lbfgs/) is used, though this can be changed. Basic usage is: - -```julia -# Generate a MLE estimate. -mle_estimate = optimize(model, MLE()) - -# Generate a MAP estimate. -map_estimate = optimize(model, MAP()) -``` - -If you wish to change to a different optimizer, such as `NelderMead`, simply place your optimizer in the third argument slot: - -```julia -# Use NelderMead -mle_estimate = optimize(model, MLE(), NelderMead()) - -# Use SimulatedAnnealing -mle_estimate = optimize(model, MLE(), SimulatedAnnealing()) - -# Use ParticleSwarm -mle_estimate = optimize(model, MLE(), ParticleSwarm()) - -# Use Newton -mle_estimate = optimize(model, MLE(), Newton()) - -# Use AcceleratedGradientDescent -mle_estimate = optimize(model, MLE(), AcceleratedGradientDescent()) -``` - -Some methods may have trouble calculating the mode because not enough iterations were allowed, or the target function moved upwards between function calls. Turing will warn you if Optim fails to converge by running `Optim.converge`. A typical solution to this might be to add more iterations, or allow the optimizer to increase between function iterations: - -```julia -# Increase the iterations and allow function eval to increase between calls. -mle_estimate = optimize(model, MLE(), Newton(), Optim.Options(iterations=10_000, allow_f_increases=true)) -``` - -More options for Optim are available [here](https://julianlsolvers.github.io/Optim.jl/stable/#user/config/). - -#### Analyzing your mode estimate - -Turing extends several methods from `StatsBase` that can be used to analyze your mode estimation results. Methods implemented include `vcov`, `informationmatrix`, `coeftable`, `params`, and `coef`, among others. - -For example, let's examine our ML estimate from above using `coeftable`: - -```julia -# Import StatsBase to use it's statistical methods. -using StatsBase - -# Print out the coefficient table. -coeftable(mle_estimate) -``` - -```julia -───────────────────────────── - estimate stderror tstat -───────────────────────────── -s 0.0625 0.0625 1.0 -m 1.75 0.176777 9.8995 -───────────────────────────── -``` - -Standard errors are calculated from the Fisher information matrix (inverse Hessian of the log likelihood or log joint). t-statistics will be familiar to frequentist statisticians. Warning -- standard errors calculated in this way may not always be appropriate for MAP estimates, so please be cautious in interpreting them. - -#### Sampling with the MAP/MLE as initial states - -You can begin sampling your chain from an MLE/MAP estimate by extracting the vector of parameter values and providing it to the `sample` function with the keyword `init_theta`. For example, here is how to sample from the full posterior using the MAP estimate as the starting point: - -```julia -# Generate an MAP estimate. -map_estimate = optimize(model, MAP()) - -# Sample with the MAP estimate as the starting point. -chain = sample(model, NUTS(), 1_000, init_theta = map_estimate.values.array) -``` - -## Beyond the Basics - - -### Compositional Sampling Using Gibbs - - -Turing.jl provides a Gibbs interface to combine different samplers. For example, one can combine an `HMC` sampler with a `PG` sampler to run inference for different parameters in a single model as below. - - -```julia -@model function simple_choice(xs) - p ~ Beta(2, 2) - z ~ Bernoulli(p) - for i in 1:length(xs) - if z == 1 - xs[i] ~ Normal(0, 1) - else - xs[i] ~ Normal(2, 1) - end - end -end - -simple_choice_f = simple_choice([1.5, 2.0, 0.3]) - -chn = sample(simple_choice_f, Gibbs(HMC(0.2, 3, :p), PG(20, :z)), 1000) -``` - - -The `Gibbs` sampler can be used to specify unique automatic differentation backends for different variable spaces. Please see the [Automatic Differentiation]({{site.baseurl}}/docs/using-turing/autodiff) article for more. - - -For more details of compositional sampling in Turing.jl, please check the corresponding [paper](http://proceedings.mlr.press/v84/ge18b.html). - - -### Working with MCMCChains.jl - - -Turing.jl wraps its samples using `MCMCChains.Chain` so that all the functions working for `MCMCChains.Chain` can be re-used in Turing.jl. Two typical functions are `MCMCChains.describe` and `MCMCChains.plot`, which can be used as follows for an obtained chain `chn`. For more information on `MCMCChains`, please see the [GitHub repository](https://github.com/TuringLang/MCMCChains.jl). - - -```julia -describe(chn) # Lists statistics of the samples. -plot(chn) # Plots statistics of the samples. -``` - - -There are numerous functions in addition to `describe` and `plot` in the `MCMCChains` package, such as those used in convergence diagnostics. For more information on the package, please see the [GitHub repository](https://github.com/TuringLang/MCMCChains.jl). - - -### Working with Libtask.jl - - -The [Libtask.jl](https://github.com/TuringLang/Libtask.jl) library provides write-on-copy data structures that are safe for use in Turing's particle-based samplers. One data structure in particular is often required for use – the [`TArray`](http://turing.ml/docs/library/#Libtask.TArray). The following sampler types require the use of a `TArray` to store distributions: - - - * `IPMCMC` - * `IS` - * `PG` - * `PMMH` - * `SMC` - - -If you do not use a `TArray` to store arrays of distributions when using a particle-based sampler, you may experience errors. - - -Here is an example of how the `TArray` (using a `TArray` constructor function called `tzeros`) can be applied in this way: - - -```julia -# Turing model definition. -@model function BayesHmm(y) - # Declare a TArray with a length of N. - s = tzeros(Int, N) - m = Vector{Real}(undef, K) - T = Vector{Vector{Real}}(undef, K) - for i = 1:K - T[i] ~ Dirichlet(ones(K)/K) - m[i] ~ Normal(i, 0.01) - end - - # Draw from a distribution for each element in s. - s[1] ~ Categorical(K) - for i = 2:N - s[i] ~ Categorical(vec(T[s[i-1]])) - y[i] ~ Normal(m[s[i]], 0.1) - end - return (s, m) -end; -``` - - -### Changing Default Settings - - -Some of Turing.jl's default settings can be changed for better usage. - - -#### AD Chunk Size - - -ForwardDiff (Turing's default AD backend) uses forward-mode chunk-wise AD. The chunk size can be manually set by `setchunksize(new_chunk_size)`; alternatively, use an auto-tuning helper function `auto_tune_chunk_size!(mf::Function, rep_num=10)`, which will profile various chunk sizes. Here `mf` is the model function, e.g. `gdemo(1.5, 2)`, and `rep_num` is the number of repetitions during profiling. - - -#### AD Backend - - -Turing supports four packages of automatic differentiation (AD) in the back end during sampling. The default AD backend is [ForwardDiff](https://github.com/JuliaDiff/ForwardDiff.jl) for forward-mode AD. Three reverse-mode AD backends are also supported, namely [Tracker](https://github.com/FluxML/Tracker.jl), [Zygote](https://github.com/FluxML/Zygote.jl) and [ReverseDiff](https://github.com/JuliaDiff/ReverseDiff.jl). `Zygote` and `ReverseDiff` are supported optionally if explicitly loaded by the user with `using Zygote` or `using ReverseDiff` next to `using Turing`. - -For more information on Turing's automatic differentiation backend, please see the [Automatic Differentiation]({{site.baseurl}}/docs/using-turing/autodiff) article. - - -#### Progress Logging - -Turing.jl uses ProgressLogging.jl to log the progress of sampling. Progress -logging is enabled as default but might slow down inference. It can be turned on -or off by setting the keyword argument `progress` of `sample` to `true` or `false`, respectively. Moreover, you can enable or disable progress logging globally by calling `turnprogress(true)` or `turnprogress(false)`, respectively. - -Turing uses heuristics to select an appropriate visualization backend. If you -use [Juno](https://junolab.org/), the progress is displayed with a -[progress bar in the Atom window](http://docs.junolab.org/latest/man/juno_frontend/#Progress-Meters-1). -For Jupyter notebooks the default backend is -[ConsoleProgressMonitor.jl](https://github.com/tkf/ConsoleProgressMonitor.jl). -In all other cases progress logs are displayed with -[TerminalLoggers.jl](https://github.com/c42f/TerminalLoggers.jl). Alternatively, -if you provide a custom visualization backend, Turing uses it instead of the -default backend. diff --git a/src/using-turing/index.md b/src/using-turing/index.md deleted file mode 100644 index e70dd8c6e..000000000 --- a/src/using-turing/index.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: Turing Documentation ---- - -# Turing Documentation - -Welcome to the documentation for Turing. - - -## Introduction - -**Turing** is a general-purpose probabilistic programming language for robust, efficient Bayesian inference and decision making. Current features include: - - - * **General-purpose** probabilistic programming with an intuitive modelling interface; - * Robust, efficient [Hamiltonian Monte Carlo (HMC)](https://github.com/TuringLang/AdvancedHMC.jl) sampling for differentiable posterior distributions; - * **Particle MCMC** sampling for complex posterior distributions involving discrete variables and stochastic control flow; and - * Compositional inference via **Gibbs sampling** that combines particle MCMC, HMC and random-walk MH (RWMH). - diff --git a/src/using-turing/performancetips.md b/src/using-turing/performancetips.md deleted file mode 100644 index 075909b1b..000000000 --- a/src/using-turing/performancetips.md +++ /dev/null @@ -1,160 +0,0 @@ ---- -title: Performance Tips ---- - -# Performance Tips - -This section briefly summarises a few common techniques to ensure good performance when using Turing. -We refer to [the Julia documentation](https://docs.julialang.org/en/v1/manual/performance-tips/index.html) for general techniques to ensure good performance of Julia programs. - - -## Use multivariate distributions - -It is generally preferable to use multivariate distributions if possible. - -The following example: - -```julia -@model function gmodel(x) - m ~ Normal() - for i = 1:length(x) - x[i] ~ Normal(m, 0.2) - end -end -``` -can be directly expressed more efficiently using a simple transformation: - -```julia -using FillArrays - -@model function gmodel(x) - m ~ Normal() - x ~ MvNormal(Fill(m, length(x)), 0.2) -end -``` - - -## Choose your AD backend -Turing currently provides support for two different automatic differentiation (AD) backends. -Generally, try to use `:forwarddiff` for models with few parameters and `:reversediff`, `:tracker` or `:zygote` for models with large parameter vectors or linear algebra operations. See [Automatic Differentiation](autodiff) for details. - - -## Special care for `:tracker` and `:zygote` - -In case of `:tracker` and `:zygote`, it is necessary to avoid loops for now. -This is mainly due to the reverse-mode AD backends `Tracker` and `Zygote` which are inefficient for such cases. `ReverseDiff` does better but vectorized operations will still perform better. - -Avoiding loops can be done using `filldist(dist, N)` and `arraydist(dists)`. `filldist(dist, N)` creates a multivariate distribution that is composed of `N` identical and independent copies of the univariate distribution `dist` if `dist` is univariate, or it creates a matrix-variate distribution composed of `N` identical and idependent copies of the multivariate distribution `dist` if `dist` is multivariate. `filldist(dist, N, M)` can also be used to create a matrix-variate distribution from a univariate distribution `dist`. `arraydist(dists)` is similar to `filldist` but it takes an array of distributions `dists` as input. Writing a [custom distribution](advanced) with a custom adjoint is another option to avoid loops. - - -## Ensure that types in your model can be inferred - -For efficient gradient-based inference, e.g. using HMC, NUTS or ADVI, it is important to ensure the types in your model can be inferred. - -The following example with abstract types - -```julia -@model function tmodel(x, y) - p,n = size(x) - params = Vector{Real}(undef, n) - for i = 1:n - params[i] ~ truncated(Normal(), 0, Inf) - end - - a = x * params - y ~ MvNormal(a, 1.0) -end -``` - -can be transformed into the following representation with concrete types: - -```julia -@model function tmodel(x, y, ::Type{T} = Float64) where {T} - p,n = size(x) - params = Vector{T}(undef, n) - for i = 1:n - params[i] ~ truncated(Normal(), 0, Inf) - end - - a = x * params - y ~ MvNormal(a, 1.0) -end -``` - -Alternatively, you could use `filldist` in this example: - -```julia -@model function tmodel(x, y) - params = filldist(truncated(Normal(), 0, Inf), size(x, 2)) - a = x * params - y ~ MvNormal(a, 1.0) -end -``` - -Note that you can use `@code_warntype` to find types in your model definition that the compiler cannot infer. -They are marked in red in the Julia REPL. - -For example, consider the following simple program: - -```julia -@model function tmodel(x) - p = Vector{Real}(undef, 1) - p[1] ~ Normal() - p = p .+ 1 - x ~ Normal(p[1]) -end -``` - -We can use - -```julia -using Random - -model = tmodel(1.0) - -@code_warntype model.f( - Random.GLOBAL_RNG, - model, - Turing.VarInfo(model), - Turing.SampleFromPrior(), - Turing.DefaultContext(), - model.args..., -) -``` - -to inspect type inference in the model. - - -## Reuse Computations in Gibbs Sampling - -Often when performing Gibbs sampling, one can save computational time by caching the output of expensive functions. The cached values can then be reused in future Gibbs sub-iterations which do not change the inputs to this expensive function. For example in the following model -```julia -@model function demo(x) - a ~ Gamma() - b ~ Normal() - c = function1(a) - d = function2(b) - x .~ Normal(c, d) -end -alg = Gibbs(MH(:a), MH(:b)) -sample(demo(zeros(10)), alg, 1000) -``` -when only updating `a` in a Gibbs sub-iteration, keeping `b` the same, the value of `d` doesn't change. And when only updating `b`, the value of `c` doesn't change. However, if `function1` and `function2` are expensive and are both run in every Gibbs sub-iteration, a lot of time would be spent computing values that we already computed before. Such a problem can be overcome using `Memoization.jl`. Memoizing a function lets us store and reuse the output of the function for every input it is called with. This has a slight time overhead but for expensive functions, the savings will be far greater. - -To use `Memoization.jl`, simply define memoized versions of `function1` and `function2` as such: -```julia -using Memoization - -@memoize memoized_function1(args...) = function1(args...) -@memoize memoized_function2(args...) = function2(args...) -``` -Then define the `Turing` model using the new functions as such: -```julia -@model function demo(x) - a ~ Gamma() - b ~ Normal() - c = memoized_function1(a) - d = memoized_function2(b) - x .~ Normal(c, d) -end -``` diff --git a/src/using-turing/quick-start.md b/src/using-turing/quick-start.md deleted file mode 100644 index 3343e6482..000000000 --- a/src/using-turing/quick-start.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: Probablistic Programming in Thirty Seconds ---- - -# Probablistic Programming in Thirty Seconds - -If you are already well-versed in probabalistic programming and just want to take a quick look at how Turing's syntax works or otherwise just want a model to start with, we have provided a Bayesian coin-flipping model to play with. - - -This example can be run on however you have Julia installed (see [Getting Started]({{site.baseurl}}/docs/using-turing/get-started)), but you will need to install the packages `Turing` and `StatsPlots` if you have not done so already. - - -This is an excerpt from a more formal example introducing probabalistic programming which can be found in Jupyter notebook form [here](https://nbviewer.jupyter.org/github/TuringLang/TuringTutorials/blob/master/0_Introduction.ipynb) or as part of the documentation website [here]({{site.baseurl}}/tutorials). - - -```julia -# Import libraries. -using Turing, StatsPlots, Random - -# Set the true probability of heads in a coin. -p_true = 0.5 - -# Iterate from having seen 0 observations to 100 observations. -Ns = 0:100 - -# Draw data from a Bernoulli distribution, i.e. draw heads or tails. -Random.seed!(12) -data = rand(Bernoulli(p_true), last(Ns)) - -# Declare our Turing model. -@model function coinflip(y) - # Our prior belief about the probability of heads in a coin. - p ~ Beta(1, 1) - - # The number of observations. - N = length(y) - for n in 1:N - # Heads or tails of a coin are drawn from a Bernoulli distribution. - y[n] ~ Bernoulli(p) - end -end - -# Settings of the Hamiltonian Monte Carlo (HMC) sampler. -iterations = 1000 -ϵ = 0.05 -τ = 10 - -# Start sampling. -chain = sample(coinflip(data), HMC(ϵ, τ), iterations) - -# Plot a summary of the sampling process for the parameter p, i.e. the probability of heads in a coin. -histogram(chain[:p]) -``` diff --git a/src/using-turing/sampler-figs/samplers-1.svg b/src/using-turing/sampler-figs/samplers-1.svg deleted file mode 100644 index c32b05a16..000000000 --- a/src/using-turing/sampler-figs/samplers-1.svg +++ /dev/null @@ -1,2725 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 - - -2 - - -3 - - -4 - - --5 - - -0 - - -5 - - -10 - - --300 - - --200 - - --100 - - -0 - - -Gibbs{HMC, PG} - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-figs/samplers-2.svg b/src/using-turing/sampler-figs/samplers-2.svg deleted file mode 100644 index 77b377629..000000000 --- a/src/using-turing/sampler-figs/samplers-2.svg +++ /dev/null @@ -1,2479 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -4.0 - - -4.5 - - --6.5 - - --6.0 - - --5.5 - - --5.0 - - --4.5 - - --4.0 - - --3.5 - - --80 - - --70 - - --60 - - --50 - - --40 - - --30 - - -HMC - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-figs/samplers-3.svg b/src/using-turing/sampler-figs/samplers-3.svg deleted file mode 100644 index 76db8c4dc..000000000 --- a/src/using-turing/sampler-figs/samplers-3.svg +++ /dev/null @@ -1,2516 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1.5 - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -4.0 - - -4.5 - - --1 - - -0 - - -1 - - -2 - - -3 - - --125 - - --100 - - --75 - - --50 - - --25 - - -HMCDA - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-figs/samplers-4.svg b/src/using-turing/sampler-figs/samplers-4.svg deleted file mode 100644 index 08ecc3f65..000000000 --- a/src/using-turing/sampler-figs/samplers-4.svg +++ /dev/null @@ -1,2635 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0.5 - - -1.0 - - -1.5 - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - --1 - - -0 - - -1 - - -2 - - -3 - - --175 - - --150 - - --125 - - --100 - - --75 - - --50 - - --25 - - -MH() - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-figs/samplers-5.svg b/src/using-turing/sampler-figs/samplers-5.svg deleted file mode 100644 index c656086a6..000000000 --- a/src/using-turing/sampler-figs/samplers-5.svg +++ /dev/null @@ -1,2426 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1.5 - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -4.0 - - -4.5 - - --1 - - -0 - - -1 - - -2 - - -3 - - -4 - - --80 - - --70 - - --60 - - --50 - - --40 - - --30 - - --20 - - -NUTS(0.65) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-figs/samplers-6.svg b/src/using-turing/sampler-figs/samplers-6.svg deleted file mode 100644 index 76badce95..000000000 --- a/src/using-turing/sampler-figs/samplers-6.svg +++ /dev/null @@ -1,2222 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -2 - - -3 - - -4 - - -5 - - --7 - - --6 - - --5 - - --4 - - --80 - - --70 - - --60 - - --50 - - --40 - - --30 - - -NUTS(0.95) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-figs/samplers-7.svg b/src/using-turing/sampler-figs/samplers-7.svg deleted file mode 100644 index c0dc125bc..000000000 --- a/src/using-turing/sampler-figs/samplers-7.svg +++ /dev/null @@ -1,2562 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -2.0 - - -2.5 - - -3.0 - - -3.5 - - -4.0 - - -4.5 - - -0 - - -2 - - -4 - - -6 - - --70 - - --60 - - --50 - - --40 - - --30 - - --20 - - -NUTS(0.2) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-figs/samplers-8.svg b/src/using-turing/sampler-figs/samplers-8.svg deleted file mode 100644 index 4382ee70e..000000000 --- a/src/using-turing/sampler-figs/samplers-8.svg +++ /dev/null @@ -1,2691 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -1 - - -2 - - -3 - - -4 - - --5 - - -0 - - -5 - - -10 - - --400 - - --300 - - --200 - - --100 - - -0 - - -PG(20) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-figs/samplers-9.svg b/src/using-turing/sampler-figs/samplers-9.svg deleted file mode 100644 index baefbb519..000000000 --- a/src/using-turing/sampler-figs/samplers-9.svg +++ /dev/null @@ -1,2616 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -0 - - -1 - - -2 - - -3 - - -4 - - --5 - - -0 - - -5 - - -10 - - --800 - - --600 - - --400 - - --200 - - -0 - - -PG(50) - - -σ - - -μ - - -Log probability - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/using-turing/sampler-viz.md b/src/using-turing/sampler-viz.md deleted file mode 100644 index 324428256..000000000 --- a/src/using-turing/sampler-viz.md +++ /dev/null @@ -1,212 +0,0 @@ ---- -title: Sampler Visualization -mathjax: true ---- - -# Sampler Visualization - -## Introduction - -## The Code - -For each sampler, we will use the same code to plot sampler paths. The block below loads the relevant libraries and defines a function for plotting the sampler's trajectory across the posterior. - -The Turing model definition used here is not especially practical, but it is designed in such a way as to produce visually interesting posterior surfaces to show how different samplers move along the distribution. - -```julia -ENV["GKS_ENCODING"] = "utf-8" # Allows the use of unicode characters in Plots.jl -using Plots -using StatsPlots -using Turing -using Bijectors -using Random -using DynamicPPL: getlogp, settrans!, getval, reconstruct, vectorize, setval! - -# Set a seed. -Random.seed!(0) - -# Define a strange model. -@model gdemo(x) = begin - s ~ InverseGamma(2, 3) - m ~ Normal(0, sqrt(s)) - bumps = sin(m) + cos(m) - m = m + 5*bumps - for i in eachindex(x) - x[i] ~ Normal(m, sqrt(s)) - end - return s, m -end - -# Define our data points. -x = [1.5, 2.0, 13.0, 2.1, 0.0] - -# Set up the model call, sample from the prior. -model = gdemo(x) -vi = Turing.VarInfo(model) - -# Convert the variance parameter to the real line before sampling. -# Note: We only have to do this here because we are being very hands-on. -# Turing will handle all of this for you during normal sampling. -dist = InverseGamma(2,3) -svn = vi.metadata.s.vns[1] -mvn = vi.metadata.m.vns[1] -setval!(vi, vectorize(dist, Bijectors.link(dist, reconstruct(dist, getval(vi, svn)))), svn) -settrans!(vi, true, svn) - -# Evaluate surface at coordinates. -function evaluate(m1, m2) - spl = Turing.SampleFromPrior() - vi[svn] = [m1] - vi[mvn] = [m2] - model(vi, spl) - getlogp(vi) -end - -function plot_sampler(chain; label="") - # Extract values from chain. - val = get(chain, [:s, :m, :lp]) - ss = link.(Ref(InverseGamma(2, 3)), val.s) - ms = val.m - lps = val.lp - - # How many surface points to sample. - granularity = 100 - - # Range start/stop points. - spread = 0.5 - σ_start = minimum(ss) - spread * std(ss); σ_stop = maximum(ss) + spread * std(ss); - μ_start = minimum(ms) - spread * std(ms); μ_stop = maximum(ms) + spread * std(ms); - σ_rng = collect(range(σ_start, stop=σ_stop, length=granularity)) - μ_rng = collect(range(μ_start, stop=μ_stop, length=granularity)) - - # Make surface plot. - p = surface(σ_rng, μ_rng, evaluate, - camera=(30, 65), - # ticks=nothing, - colorbar=false, - color=:inferno, - title=label) - - line_range = 1:length(ms) - - scatter3d!(ss[line_range], ms[line_range], lps[line_range], - mc =:viridis, marker_z=collect(line_range), msw=0, - legend=false, colorbar=false, alpha=0.5, - xlabel="σ", ylabel="μ", zlabel="Log probability", - title=label) - - return p -end; -``` - -## Samplers - -### Gibbs - -Gibbs sampling tends to exhibit a "jittery" trajectory. The example below combines `HMC` and `PG` sampling to traverse the posterior. - -```julia -c = sample(model, Gibbs(HMC(0.01, 5, :s), PG(20, :m)), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-1.svg) - -### HMC - -Hamiltonian Monte Carlo (HMC) sampling is a typical sampler to use, as it tends to be fairly good at converging in a efficient manner. It can often be tricky to set the correct parameters for this sampler however, and the `NUTS` sampler is often easier to run if you don't want to spend too much time fiddling with step size and and the number of steps to take. Note however that `HMC` does not explore the positive values μ very well, likely due to the leapfrop and step size parameter settings. - -```julia -c = sample(model, HMC(0.01, 10), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-2.svg) - - -### HMCDA - -The HMCDA sampler is an implementation of the Hamiltonian Monte Carlo with Dual Averaging algorithm found in the paper "The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo" by Hoffman and Gelman (2011). The paper can be found on [arXiv](https://arxiv.org/abs/1111.4246) for the interested reader. - -```julia -c = sample(model, HMCDA(200, 0.65, 0.3), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-3.svg) - - -### MH - -Metropolis-Hastings (MH) sampling is one of the earliest Markov Chain Monte Carlo methods. MH sampling does not "move" a lot, unlike many of the other samplers implemented in Turing. Typically a much longer chain is required to converge to an appropriate parameter estimate. - -The plot below only uses 1,000 iterations of Metropolis-Hastings. - -```julia -c = sample(model, MH(), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-4.svg) - - -As you can see, the MH sampler doesn't move parameter estimates very often. - -### NUTS - -The No U-Turn Sampler (NUTS) is an implementation of the algorithm found in the paper "The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo" by Hoffman and Gelman (2011). The paper can be found on [arXiv](https://arxiv.org/abs/1111.4246) for the interested reader. - -NUTS tends to be very good at traversing complex posteriors quickly. - -```julia -c = sample(model, NUTS(0.65), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-5.svg) - - -The only parameter that needs to be set other than the number of iterations to run is the target acceptance rate. In the Hoffman and Gelman paper, they note that a target acceptance rate of 0.65 is typical. - -Here is a plot showing a very high acceptance rate. Note that it appears to "stick" to a mode and is not particularly good at exploring the posterior as compared to the 0.65 target acceptance ratio case. - -```julia -c = sample(model, NUTS(0.95), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-6.svg) - - -An exceptionally low acceptance rate will show very few moves on the posterior: - -```julia -c = sample(model, NUTS(0.2), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-7.svg) - - -### PG - -The Particle Gibbs (PG) sampler is an implementation of an algorithm from the paper "Particle Markov chain Monte Carlo methods" by Andrieu, Doucet, and Holenstein (2010). The interested reader can learn more [here](https://rss.onlinelibrary.wiley.com/doi/full/10.1111/j.1467-9868.2009.00736.x). - -The two parameters are the number of particles, and the number of iterations. The plot below shows the use of 20 particles. - -```julia -c = sample(model, PG(20), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-8.svg) - - -Next, we plot using 50 particles. - -```julia -c = sample(model, PG(50), 1000) -plot_sampler(c) -``` - -![](sampler-figs/samplers-9.svg)