JuliaFEM.jl/notebooks/2015-06-16-iterative-solvers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Iterative solvers, numerical study of convergence\n",
    "\n",
    "Author(s): Jukka.Aho <jukka.aho@kapsi.fi>\n",
    "\n",
    "##Abstract\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO: Loading help data...\n"
     ]
    }
   ],
   "source": [
    "using PyPlot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "endpoint of true solution: 0.5000000000000506\n",
      "Reference norm: 3.182045987352757\n"
     ]
    }
   ],
   "source": [
    "N = 200\n",
    "A = N*full(Tridiagonal(-ones(N-1), 2*ones(N), -1*ones(N-1)))\n",
    "A[end,end] -= N\n",
    "b = -1/N*ones(N)\n",
    "b[end] += 1 + 1/(2*N)\n",
    "x_true = A\\b\n",
    "println(\"endpoint of true solution: \", x_true[end])\n",
    "rnorm = norm(A\\b, 2)\n",
    "println(\"Reference norm: \", rnorm)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Jacobi method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "function solve_Jacobi(A, b, max_iterations=3000)\n",
    "    N = length(b)\n",
    "    R = A-diagm(diag(A))\n",
    "    invD = diagm(1./diag(A))\n",
    "    x = zeros(N)\n",
    "    norms = []\n",
    "    for i=1:max_iterations\n",
    "        x = invD*(b-R*x)\n",
    "        norms = [norms; norm(x)]\n",
    "    end\n",
    "    return x, norms\n",
    "end\n",
    "x_jacobi, norms_jacobi = solve_Jacobi(A, b);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Richardson method\n",
    "\n",
    "- is equivalent with deepest descent method for min 1/2*x'*A*x - b'x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "function solve_Richardson(A, b, max_iterations=3000)\n",
    "    N = length(b)\n",
    "    x = zeros(N)\n",
    "    norms = []\n",
    "    la = sort(eigvals(A), rev=true)\n",
    "    wopt = 2/(maximum(la)+minimum(la))\n",
    "    # modification, remove wave of smallest and largest eigenvalue because they are anyway calculated.\n",
    "    # this increases convergence a bit\n",
    "    for i = 1:1\n",
    "        x = x - 1/la[i]*(A*x - b)\n",
    "        x = x - 1/la[end-i]*(A*x - b)\n",
    "    end\n",
    "    for i=1:max_iterations\n",
    "        x = x - wopt*(A*x - b)\n",
    "        norms = [norms; norm(x)]\n",
    "    end\n",
    "    return x, norms\n",
    "end\n",
    "\n",
    "x_richardson, norms_richardson = solve_Richardson(A, b);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conjugate gradient\n",
    "## Successive over-relaxation\n",
    "## GMRES\n",
    "## MINRES"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAArsAAAI6CAYAAAAnj4D/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XlYVnX6P/D3edgfERBccAk3HLS0FFwC9xw1NZ8s91zByTHFUksnbUaxsVLsV6aY1eSappam2aihOSqS2hBIlopBqWjuqLiggnB+fzjw5VmOIgLn5vB+XRfXpWe9H96WNx8/53MUVVVVEBEREREZkEnvAoiIiIiISgubXSIiIiIyLDa7RERERGRYbHaJiIiIyLDY7BIRERGRYbHZJSIiIiLDYrNLRERERIbFZpeIiIiIDIvNLhEREREZFptdIiIiIjIsNrtEJFpKSgrGjx+Ppk2bwtvbG25ubqhduzaeeeYZLFmyBNnZ2XqXSEREgimqqqp6F0FE5Mibb76JmTNnQlVVhIWFoWXLlqhcuTLOnj2LuLg4pKamIiQkBAkJCXqXSkREQjnrXQARkSNvv/02oqKiEBAQgC+//BKtWrWyOyY2NhZz587VoToiIiovOI2BiMQ5fvw4oqKi4Orqii1btjhsdAGge/fu2LJli9W2L774Ah06dIC3tzfMZjMef/xxzJ492+F0h3r16qF+/frIysrC5MmTERAQAHd3dzRq1AjR0dFWx+7fvx8mkwnPP/+8Zt1NmjSBu7s7rly5YrU9NjYWPXv2RNWqVeHu7o7AwEBMmTIFmZmZmjVdu3YNkyZNQr169eDq6oqZM2daXa9t27aoVKkS/Pz88NxzzyElJQUjR46EyWRCenq63XV/+OEH9OvXD/7+/nBzc0NAQADGjBmDM2fO2B3bqVMnmEwm5Obm4u2330ajRo3g7u6OgIAAvP7668jJyXH4+VNSUhAREYF69erB3d0dNWrUQIcOHfDRRx85PHbkyJF45JFH4ObmBn9/fwwZMgS//vqr5veXiKg4OLJLROIsXboUd+7cweDBg/Hoo4/e81hXV9eCX0+bNg2zZ89GtWrVMHToUHh6emLLli2YNm0aYmNjsW3bNri4uBQcrygKcnJy0K1bN5w5cwa9evWCs7MzNmzYgNdffx23bt3C9OnTAQBPPvkkgoKCsGXLFly6dAm+vr5Wdfz3v//F0aNH0a9fP/j4+BRsnzlzJmbOnAk/Pz/07t0b1atXx08//YR3330XW7Zswb59+1C5cmWrmrKzs9G5c2dcuXIFTz/9NLy8vNCgQQMAwJo1a/DCCy/AbDZj4MCBqFmzJr7//nuEhYXhiSeecPg9WrJkCUaPHg0PDw9YLBY88sgj+PXXX/Hpp5/im2++wf79+/HII4/YnTd48GDEx8ejZ8+e8PLywubNmxEdHY3z589jyZIlVsdu3rwZ/fv3R3Z2Nnr06IEhQ4bgypUrSE5Oxty5czFmzJiCY7/99ls8//zzyM3NRe/evREYGIiTJ0/iq6++wubNm7Fz5060aNHinrkTERWZSkQkzFNPPaUqiqIuXry4yOfs3btXVRRFrVu3rnru3LmC7Xfu3FF79+6tKoqivv3221bn1K1bV1UURe3Vq5d669atgu3nz59XfXx8VB8fHzUnJ6dg+zvvvKMqiqLGxMTY3X/s2LGqoijqv//974Jt//nPf1RFUdS2bduqmZmZVscvW7ZMVRRFnThxosOaunbtqmZlZVntu3r1qurj46O6u7urBw8etNr3+uuvq4qiqCaTST1x4kTB9qNHj6ouLi5qo0aN1NOnT1uds2PHDtXJyUl97rnnrLZ37NhRVRRFbdmypXr58uWC7Tdu3FADAwNVJycn9ezZswXbL1y4oHp5ealubm5qXFyc3ffmjz/+KPj1pUuXVB8fH7VatWrqkSNHrI775ZdfVE9PTzU4ONjuGkRExcVml4jEadKkiaooihobG1vkc/7yl7+oiqKo//rXv+z2/frrr6qTk5PaoEEDq+1169ZVTSaT+ttvv9mdM3z4cFVRFPXQoUMF206dOqU6OTmprVq1sjr29u3bqq+vr+rv76/m5uYWbO/Tp4+qKIp6+PBhhzU3b95crV69usOabJtZVVXVzz77TFUURR01apTdvuvXr6s+Pj52ze6ECRNURVHULVu2OKyhT58+qrOzs3r9+vWCbfnN7o4dO+yOnzFjhqooirp58+aCbe+++66qKIo6YcIEh/cobN68eaqiKOqHH37ocH9+vVrfMyKiB8VpDERkCElJSVAUBU899ZTdvkaNGqF27do4fvw4rl27ZjVtwNvbu2CKQGH5/6x/+fLlgm21a9dGly5dsH37dhw5cgRNmjQBAHzzzTe4fPkyJk2aBJPp/x6F2LdvH1xcXPDFF19AdbDwTXZ2Ni5cuIDLly+jSpUqBdvd3d3RrFkzu+MPHDgAAGjXrp3dvkqVKqF58+bYvXu31fZ9+/YBAHbt2oUffvjB7rzz588jNzcXR48eRXBwcMF2RVHQsmVLu+Pr1Klj933Zv38/AKBHjx52x9vKryc5ORlRUVF2+/Pn7Bb+/hIRPQw2u0QkTs2aNZGSkoJTp04V+Zz8h71q1qypec1Tp07hypUrVs1u4fm1hTk73/3fY25urtX2kSNHYvv27Vi+fDlmz54NAFi+fDkAYMSIEVbHZmRkIDc31+rhMluKouD69etWzW716tXv+Rlr1KjhcL+j7RkZGQBwz1UrFEXBjRs37LZ7eXnZbXP0fcl/IK927dqa97Ct51//+tcD10NEVBxcjYGIxGnfvj0AYMeOHUU+x9vbGwAcri5QeHv+ccX13HPPwcvLCytXroSqqjh//jy2bt2K5s2b243Gent7w9fXF3l5eZpfubm5dg+HKYri8N75zee5c+cc7ne03dvbG4qi4OrVq/esIf97Xhz5PzAU5YeT/O//wYMH71nPsGHDil0PEVFhbHaJSJzw8HC4uLhg/fr1OHLkyD2PzV9SLDg4GKqqYteuXXbHpKWl4dSpU6hfv77D0coH4e7ujgEDBuD06dPYvn07Pv/8c+Tm5tqN6gJAaGgoLl26hMOHDz/UPfPlTzPYs2eP3b7r168jOTnZrlEODQ2FqqqIi4srkRocCQ0NBQBs3bq1yMeWZj1ERIWx2SUicerWrYuoqChkZ2ejV69eSExMdHjc1q1b8fTTTwMAIiIiAACzZs3CxYsXC47Jzc3Fa6+9BlVVMWrUqBKpb+TIkQCAFStWYMWKFXBxccGQIUPsjps4cSIA4MUXX3Q44nzjxg2H82i1PPvss/D29saqVatw8OBBq32zZs1yuG5vZGQkXFxcMHHiRKSmptrtz87Odtg8P4gRI0bAy8sLixYtcnitwiO+4eHh8PHxwcyZMx2++S4vL8/hDyxERMXFObtEJNLUqVNx584dzJw5E61atUJYWBhCQkLg6emJc+fOIS4uDmlpaQUvnAgNDcWUKVMQHR2Npk2bol+/fjCbzdi6dSsOHTqE9u3bY/LkySVSW1hYGAIDA/Hll18iJycHFosFVatWtTvuqaeewuzZszF16lQ0atQIPXv2RL169XD9+nWcOHECcXFxaN++vd2LMbRUrlwZCxcuxLBhwxAWFoYBAwbA398fe/fuxcGDB9GxY0fs3r3b6iG5oKAgLFmyBBEREXjsscfw9NNPo1GjRsjJyUF6ejr27NmDGjVq2I0+O3qgToufnx8+//xz9OvXD507d0aPHj3QrFkzXL16FQcPHsSpU6fw+++/AwB8fX2xbt06PPfcc3jyySfRpUsXPProo1AUBSdPnsS+fftw+fJlZGVlFfn+RET3pOdSEFoSExPV3r17q76+vqrZbFabNm2qzp8/X++yiEgHR44cUcePH682bdpU9fLyUl1dXdVatWqpPXv2VJcsWaJmZ2dbHb9mzRq1Xbt2auXKlVV3d3e1adOm6ttvv63evn3b7tr16tVT69ev7/C+UVFRqslkUnfv3u1w/6xZswrWtf3qq6/u+Rni4+PVAQMGqLVq1VJdXV3V6tWrqy1atFBfffVVNTExscg15du6dasaFhamms1mtUqVKmqfPn3Uo0ePqr169VIVRbFb01dVVfXnn39WR44cqdatW1d1c3NT/fz81GbNmqljxoxRd+7
      ],
      "text/plain": [
       "Figure(PyObject <matplotlib.figure.Figure object at 0x1160f6350>)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(norms_jacobi, label=\"Jacobi\")\n",
    "plot(norms_richardson, label=\"Richardson\")\n",
    "plot([0, length(norms_jacobi)], [rnorm, rnorm], label=\"True\")\n",
    "#ylim(0, 5)\n",
    "legend()\n",
    "title(\"Convergence\")\n",
    "xlabel(\"iterations N\")\n",
    "ylabel(\"L2 norm\")\n",
    "grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": [
       "iVBORw0KGgoAAAANSUhEUgAAArkAAAIhCAYAAABKa6KaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3Xt8z/X///Hb+41mGyaHHBMfhfqQOc1pheSsNzlLi1EqVIg5fD6J8iHyybk+KY1RQ2QOqWGOcwib9flUJsoaIi3HGsP2/v3x/m6/1ntj8977/dr7/b5fL5ddPl1ex/srjz4eXp6v59NktVqtiIiIiIh4ELPRAURERERECpqaXBERERHxOGpyRURERMTjqMkVEREREY+jJldEREREPI6aXBERERHxOGpyRURERMTjqMkVEREREY+jJldEREREPI6aXBGRApCUlITZbCY0NNTp9zKbzbRp08bp9xERcWdqckXEK6Snp/PBBx/QqlUrypQpw1133UWFChWoX78+zz33HBs2bCiQ+5hMJoevUb16dWrUqOH0+4iIeLKiRgcQEXG29PR0unbtSnR0NHfffTddu3alatWqXL9+nW+++YZPPvmEo0eP8sQTTxgdNcutmtjExET8/PxcmEZExP2oyRURjxcZGUl0dDSBgYHs3LmTkiVLZtt/9epVDhw4YFC6/KtVq5bREURECj0NVxARj7d3714ABg0aZNfgAvj6+tKqVSu77Wlpabz11lvUq1cPf39/AgICePTRR/n000/zfO/WrVtjNuf8f7VLlizBbDazdOlSAHbs2IHZbCY5OTlrjG/mz5/H+uY2JvfSpUtMmDCB2rVr4+vrS5kyZejYsSMxMTF2x2bea8qUKSQkJNClSxdKly6Nv78/rVu3Zt++fXl+xu7du2M2m5k/f77dvtdeew2z2cxzzz2X5+uJiBQEvckVEY9Xrlw5AI4ePZrnc65fv06HDh3YtWsXDz74ICNGjOCPP/5g9erV9O3bl4SEBP71r3/l6Vq3Gz+bub9GjRq8/vrrzJkzB4BRo0ZlHRMYGHjLa168eJGWLVty5MgRgoKC6NmzJ7/++iurVq2iffv2vPfeewwdOtTu3ocOHWLmzJm0aNGCoUOH8tNPP7FmzRratm1LQkJCnt4ah4eH06BBA8LCwnjkkUeyssbExDBt2jT+/ve/59gAi4g4lVVExMMdPnzYetddd1nNZrM1JCTE+tlnn1mTkpJuec60adOsJpPJ2qVLF2t6enrW9nPnzlmrV69uNZlM1r1792ZtP3HihNVkMllDQ0OzXadVq1ZWs9mc4z3Cw8OtJpPJunTp0mzb77vvPmuNGjVyzWYymaxt2rTJtm3o0KFWk8lkfeGFF7JtP3bsmDUgIMDq4+OT7Zm3b99uNZlMOd7//ffft5pMJuuwYcNyzfBXe/futRYrVsxaq1Yt6++//249e/astWLFilZ/f3/rd999l+friIgUFA1XEBGPFxgYyPLly6lQoQLLly+nZ8+e1KhRg7Jly9KjRw82btxod85HH32E2WzmnXfeyTbcoHz58rz22msAfPjhhy57hlu5fv06y5cvp2TJkkyfPj3bvvvvv5+XX36Z69evExERYXducHAwzzzzTLZtgwcPpkiRIhw8eDDPGZo3b86bb77JsWPHeP755wkJCeGXX35h3rx5PPjgg3f2YCIiDlCTKyJeoXfv3iQnJxMdHc2kSZOyZlKIiorCYrEwaNCgrGOvXLnCDz/8QOXKlXP86/rHHnsMgISEBJdkv52jR49y9epV6tevT+nSpe323ypv48aN7bYVLVqUChUqcOHChXzlGDduHO3bt+eTTz5h69atPPXUUwwePDhf1xARKShqckXEaxQtWpR27doxefJk1q1bx6+//srKlSvx9/cnIiKCdevWAbYPuAAqVaqU43UqVqwI2MbBFgaO5M2pKQbbv6v09PR8Z3nyyScB25jhV155Jd/ni4gUFDW5IuK1zGYzvXv3zvrAa/v27QAEBAQAcPbs2RzPO3PmTLbjbncPgIyMDLt9BdUkF2ReRxw7dowxY8Zw9913YzKZePbZZ0lLS3PqPUVEcqMmV0S8XokSJQCwWq0AlCxZkpo1a3Lq1CmOHz9ud3xmM9ywYcPbXvvuu+/GarWSnJxst+/QoUM5nlOkSJF8vUWtU6cOvr6+fP3111lvde80751KS0ujb9++XL16lVWrVjFhwgT+97//MXLkSKfdU0TkVtTkiojHi4yMZOvWrVlN7J+dPXuWDz74AIBHH300a/vgwYOxWq2MHTs221vYlJQU3nzzTUwmU57GmzZt2hQg6x6ZYmJiiIyMzPGcsmXLcu7cOa5du3b7hwOKFSvG008/zeXLl7M+isv0ww8/MG/ePO666y5CQkLydL07MWbMGBISEggLC6Nt27ZMmTKFli1b8v7777N69Wqn3VdEJDeaJ1dEPN6BAweYO3cuFStWJDg4mOrVqwNw4sQJPv/8c65du0b37t3p2bNn1jljxozhiy++YN26ddSvX59OnTqRmprKp59+SkpKCmFhYbRo0eK29w4NDeXtt99m+vTpfP311zz44IN8//33fPnll/To0YM1a9bYnfP4449z6NAhOnbsyCOPPIKPjw+BgYF07do11/u89dZb7N69mwULFnDw4EFat25NSkoKq1at4o8//mDBggXcd999+f+Xlwdr165l4cKFNGvWjKlTpwK2YRqRkZEEBgby7LPP0qhRI2rUqOGU+4uI5MjZc5Rdu3bNGhYWZq1UqZLV19fX2rRpU+uWLVvyfP6WLVusbdq0sQYEBFhLlixpbdSokXXlypVOTCwinubkyZPWhQsXWp988klr7dq1raVKlbLedddd1sqVK1u7dOli/fjjj3M879q1a9Zp06ZZ69ata/X19bWWKlXK+sgjj1hXrFhhd2xu8+RarVbrt99+a+3cubO1ZMmS1hIlSljbtGlj3bVrl3XJkiVWs9lsN0/tH3/8YX3xxRetVatWtRYtWtRqNpuzXTeneXKtVqv14sWL1nHjxlkfeOABq4+Pj/Xuu++2tm/fPsf/z82cJ3fKlCk5Pnv16tVvOVdvpp9++slapkwZ691332396aef7PavW7fOajKZrE2bNrVev379ttcTESkoJqs1h7+/K0D9+/dnzZo1jBo1igceeIDw8HAOHjzI9u3badmy5S3PDQ8P59lnn6V9+/ZYLBaKFClCYmIiVatWZfTo0c6MLSIiIiJuzKlN7oEDB2jWrBmzZs3KakrT0tKoW7cu99xzD3v27Mn13KSkJB566CGef/55Zs+e7ayIIiIiIuKBnPrh2erVqylatGi29dJ9fHwYMmQI+/bt4/Tp07me+5///Aer1cobb7wBwO+//57jRyMiIiIiIn/l1Cb38OHD1KpVK2t6nkxNmjQBbr1a0NatW6lTpw4bN26katWqlCpVinLlyjFp0iQ1uyIiIiJyS06dXeHMmTM5rsCTue3nn3/O9dxjx45RtGhRBg8ezLhx46hfvz5r1qxh6tSp3Lx5k2nTpjktt4iIiIi4N6c2uVevXsXHx8due/HixbP25yZzeMKMGTMYO3YsYFsu8vz588ydO5eJEyfavSEG2xyW0dHRVK9eHV9f3wJ6EhEREREpKFevXiUpKYkOHTpQrlw5p9zDqU2ur69vjks6Zk5wfqsm1NfXl6tXr9K/f/9s2/v168eXX35JQkICwcHBdudFR0fz9NNPO5hcRERERJxt+fLlDBgwwCnXdmqTW6lSpRyHJGSuo165cuVcz61cuTI//PADFSpUyLb9nnvuAeDChQs5npc5yfvy5ct58MEH7yS2SJ716tVLqzmJS6jWxFVUa+IKR44c4emnn87q25zBqU1ugwYN2LFjB1euXKFkyZJZ27/66isAAgMDcz23cePGHD9+nFOnTmVbJSezaS5fvnyO52W+HX7wwQeduk67CNiWU1WdiSuo1sRVVGviSs4cWurU2RV69epFeno6ixYtytqWlpZGeHg4zZo1o0qVKoBt7fjExERu3ryZdVzfvn0BWLx4cda2jIwMwsPDKVu2LI0aNXJmdJE8qV27ttERxEuo1sRVVGviKZz6JjcoKIjevXszYcIEzp07R82aNVm6dCnJycmEh4dnHTd+/HgiIiJISkq
      ],
      "text/plain": [
       "Figure(PyObject <matplotlib.figure.Figure object at 0x117a76e90>)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(x_jacobi, label=\"Jacobi\")\n",
    "plot(x_richardson, label=\"Richardon\")\n",
    "plot(A\\b, label=\"True\")\n",
    "legend(loc=\"best\")\n",
    "title(\"Solution x\")\n",
    "grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.3.8",
   "language": "julia",
   "name": "julia-0.3"
  },
  "language_info": {
   "name": "julia",
   "version": "0.3.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}