julia/gae-benchmark-results.ipynb
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# GAE benchmark results"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import numpy as np\n",
"%matplotlib inline\n",
"import pandas as pd\n",
"import matplotlib.pyplot as plt\n",
"import seaborn as sb"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## AN2VEC"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"results = pd.read_csv('../data/behaviour/gae-benchmarks/lp-bias=[true, false]-sharedl1=[true, false]-'\n",
" 'decadjdeep=[true, false]-dataset=[cora, citeseer, pubmed]-'\n",
" 'overlap=[0, 8, 16]-testprop=[0.15, 0.30]-nsamples=10.csv.gz',\n",
" compression='gzip')\n",
"last_epoch = sorted(np.unique(results.epoch))[-1]\n",
"results = results[results.epoch == last_epoch]"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"averages = results.groupby(['bias', 'sharedl1', 'decadjdeep', 'dataset', 'overlap', 'blurring']).mean()\n",
"stds = results.groupby(['bias', 'sharedl1', 'decadjdeep', 'dataset', 'overlap', 'blurring']).std()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Shared l1"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th></th>\n",
" <th>dataset</th>\n",
" <th colspan=\"2\" halign=\"left\">cora</th>\n",
" <th colspan=\"2\" halign=\"left\">citeseer</th>\n",
" <th colspan=\"2\" halign=\"left\">pubmed</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" </tr>\n",
" <tr>\n",
" <th>blurring</th>\n",
" <th>decadjdeep</th>\n",
" <th>overlap</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th rowspan=\"6\" valign=\"top\">0.15</th>\n",
" <th rowspan=\"3\" valign=\"top\">False</th>\n",
" <th>0</th>\n",
" <td>0.928669</td>\n",
" <td>0.932253</td>\n",
" <td>0.940158</td>\n",
" <td>0.945510</td>\n",
" <td>0.950613</td>\n",
" <td>0.953707</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.930294</td>\n",
" <td>0.935635</td>\n",
" <td>0.944549</td>\n",
" <td>0.948189</td>\n",
" <td>0.946433</td>\n",
" <td>0.948626</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.932945</td>\n",
" <td>0.936042</td>\n",
" <td>0.948220</td>\n",
" <td>0.951456</td>\n",
" <td>0.930713</td>\n",
" <td>0.930576</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">True</th>\n",
" <th>0</th>\n",
" <td>0.883429</td>\n",
" <td>0.894645</td>\n",
" <td>0.908322</td>\n",
" <td>0.913254</td>\n",
" <td>0.917656</td>\n",
" <td>0.931821</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.887484</td>\n",
" <td>0.896085</td>\n",
" <td>0.902848</td>\n",
" <td>0.909423</td>\n",
" <td>0.918967</td>\n",
" <td>0.930725</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.892824</td>\n",
" <td>0.903344</td>\n",
" <td>0.904231</td>\n",
" <td>0.911761</td>\n",
" <td>0.920825</td>\n",
" <td>0.927513</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"6\" valign=\"top\">0.30</th>\n",
" <th rowspan=\"3\" valign=\"top\">False</th>\n",
" <th>0</th>\n",
" <td>0.916112</td>\n",
" <td>0.924120</td>\n",
" <td>0.934943</td>\n",
" <td>0.938995</td>\n",
" <td>0.948759</td>\n",
" <td>0.950269</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.918903</td>\n",
" <td>0.926251</td>\n",
" <td>0.935658</td>\n",
" <td>0.939858</td>\n",
" <td>0.941399</td>\n",
" <td>0.943468</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.921719</td>\n",
" <td>0.928890</td>\n",
" <td>0.943262</td>\n",
" <td>0.948098</td>\n",
" <td>0.924663</td>\n",
" <td>0.923990</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">True</th>\n",
" <th>0</th>\n",
" <td>0.868274</td>\n",
" <td>0.880303</td>\n",
" <td>0.892681</td>\n",
" <td>0.900440</td>\n",
" <td>0.910215</td>\n",
" <td>0.924337</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.865211</td>\n",
" <td>0.879828</td>\n",
" <td>0.896792</td>\n",
" <td>0.904109</td>\n",
" <td>0.911458</td>\n",
" <td>0.923809</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.872229</td>\n",
" <td>0.884455</td>\n",
" <td>0.890310</td>\n",
" <td>0.895504</td>\n",
" <td>0.913918</td>\n",
" <td>0.920653</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"dataset cora citeseer pubmed \\\n",
" auc ap auc ap auc \n",
"blurring decadjdeep overlap \n",
"0.15 False 0 0.928669 0.932253 0.940158 0.945510 0.950613 \n",
" 8 0.930294 0.935635 0.944549 0.948189 0.946433 \n",
" 16 0.932945 0.936042 0.948220 0.951456 0.930713 \n",
" True 0 0.883429 0.894645 0.908322 0.913254 0.917656 \n",
" 8 0.887484 0.896085 0.902848 0.909423 0.918967 \n",
" 16 0.892824 0.903344 0.904231 0.911761 0.920825 \n",
"0.30 False 0 0.916112 0.924120 0.934943 0.938995 0.948759 \n",
" 8 0.918903 0.926251 0.935658 0.939858 0.941399 \n",
" 16 0.921719 0.928890 0.943262 0.948098 0.924663 \n",
" True 0 0.868274 0.880303 0.892681 0.900440 0.910215 \n",
" 8 0.865211 0.879828 0.896792 0.904109 0.911458 \n",
" 16 0.872229 0.884455 0.890310 0.895504 0.913918 \n",
"\n",
"dataset \n",
" ap \n",
"blurring decadjdeep overlap \n",
"0.15 False 0 0.953707 \n",
" 8 0.948626 \n",
" 16 0.930576 \n",
" True 0 0.931821 \n",
" 8 0.930725 \n",
" 16 0.927513 \n",
"0.30 False 0 0.950269 \n",
" 8 0.943468 \n",
" 16 0.923990 \n",
" True 0 0.924337 \n",
" 8 0.923809 \n",
" 16 0.920653 "
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"averages.xs([False, True])\\\n",
" [['auc', 'ap']]\\\n",
" .stack().unstack([1, 4])\\\n",
" .reorder_levels(['blurring', 'decadjdeep', 'overlap']).sort_index()\\\n",
" [['cora', 'citeseer', 'pubmed']]"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th></th>\n",
" <th>dataset</th>\n",
" <th colspan=\"2\" halign=\"left\">cora</th>\n",
" <th colspan=\"2\" halign=\"left\">citeseer</th>\n",
" <th colspan=\"2\" halign=\"left\">pubmed</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" </tr>\n",
" <tr>\n",
" <th>blurring</th>\n",
" <th>decadjdeep</th>\n",
" <th>overlap</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th rowspan=\"6\" valign=\"top\">0.15</th>\n",
" <th rowspan=\"3\" valign=\"top\">False</th>\n",
" <th>0</th>\n",
" <td>0.004995</td>\n",
" <td>0.007034</td>\n",
" <td>0.005996</td>\n",
" <td>0.005695</td>\n",
" <td>0.006221</td>\n",
" <td>0.005054</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.003856</td>\n",
" <td>0.005355</td>\n",
" <td>0.007048</td>\n",
" <td>0.007055</td>\n",
" <td>0.005213</td>\n",
" <td>0.004407</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.007901</td>\n",
" <td>0.007053</td>\n",
" <td>0.004446</td>\n",
" <td>0.004953</td>\n",
" <td>0.006167</td>\n",
" <td>0.005951</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">True</th>\n",
" <th>0</th>\n",
" <td>0.008031</td>\n",
" <td>0.008416</td>\n",
" <td>0.014037</td>\n",
" <td>0.015891</td>\n",
" <td>0.006894</td>\n",
" <td>0.005810</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.016852</td>\n",
" <td>0.018730</td>\n",
" <td>0.013327</td>\n",
" <td>0.012892</td>\n",
" <td>0.004525</td>\n",
" <td>0.004613</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.008851</td>\n",
" <td>0.010439</td>\n",
" <td>0.013957</td>\n",
" <td>0.012869</td>\n",
" <td>0.009610</td>\n",
" <td>0.009229</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"6\" valign=\"top\">0.30</th>\n",
" <th rowspan=\"3\" valign=\"top\">False</th>\n",
" <th>0</th>\n",
" <td>0.007145</td>\n",
" <td>0.006813</td>\n",
" <td>0.003441</td>\n",
" <td>0.003182</td>\n",
" <td>0.003814</td>\n",
" <td>0.003474</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.003457</td>\n",
" <td>0.003879</td>\n",
" <td>0.006418</td>\n",
" <td>0.007233</td>\n",
" <td>0.009610</td>\n",
" <td>0.009046</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.006675</td>\n",
" <td>0.006816</td>\n",
" <td>0.006928</td>\n",
" <td>0.007518</td>\n",
" <td>0.006093</td>\n",
" <td>0.007091</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">True</th>\n",
" <th>0</th>\n",
" <td>0.012494</td>\n",
" <td>0.011262</td>\n",
" <td>0.012345</td>\n",
" <td>0.011420</td>\n",
" <td>0.003784</td>\n",
" <td>0.003233</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.011779</td>\n",
" <td>0.010993</td>\n",
" <td>0.011018</td>\n",
" <td>0.010990</td>\n",
" <td>0.005377</td>\n",
" <td>0.004543</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.010179</td>\n",
" <td>0.011519</td>\n",
" <td>0.024513</td>\n",
" <td>0.025013</td>\n",
" <td>0.006671</td>\n",
" <td>0.006568</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"dataset cora citeseer pubmed \\\n",
" auc ap auc ap auc \n",
"blurring decadjdeep overlap \n",
"0.15 False 0 0.004995 0.007034 0.005996 0.005695 0.006221 \n",
" 8 0.003856 0.005355 0.007048 0.007055 0.005213 \n",
" 16 0.007901 0.007053 0.004446 0.004953 0.006167 \n",
" True 0 0.008031 0.008416 0.014037 0.015891 0.006894 \n",
" 8 0.016852 0.018730 0.013327 0.012892 0.004525 \n",
" 16 0.008851 0.010439 0.013957 0.012869 0.009610 \n",
"0.30 False 0 0.007145 0.006813 0.003441 0.003182 0.003814 \n",
" 8 0.003457 0.003879 0.006418 0.007233 0.009610 \n",
" 16 0.006675 0.006816 0.006928 0.007518 0.006093 \n",
" True 0 0.012494 0.011262 0.012345 0.011420 0.003784 \n",
" 8 0.011779 0.010993 0.011018 0.010990 0.005377 \n",
" 16 0.010179 0.011519 0.024513 0.025013 0.006671 \n",
"\n",
"dataset \n",
" ap \n",
"blurring decadjdeep overlap \n",
"0.15 False 0 0.005054 \n",
" 8 0.004407 \n",
" 16 0.005951 \n",
" True 0 0.005810 \n",
" 8 0.004613 \n",
" 16 0.009229 \n",
"0.30 False 0 0.003474 \n",
" 8 0.009046 \n",
" 16 0.007091 \n",
" True 0 0.003233 \n",
" 8 0.004543 \n",
" 16 0.006568 "
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"stds.xs([False, True])\\\n",
" [['auc', 'ap']]\\\n",
" .stack().unstack([1, 4])\\\n",
" .reorder_levels(['blurring', 'decadjdeep', 'overlap']).sort_index()\\\n",
" [['cora', 'citeseer', 'pubmed']]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Unshared l1"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th></th>\n",
" <th>dataset</th>\n",
" <th colspan=\"2\" halign=\"left\">cora</th>\n",
" <th colspan=\"2\" halign=\"left\">citeseer</th>\n",
" <th colspan=\"2\" halign=\"left\">pubmed</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" </tr>\n",
" <tr>\n",
" <th>blurring</th>\n",
" <th>decadjdeep</th>\n",
" <th>overlap</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th rowspan=\"6\" valign=\"top\">0.15</th>\n",
" <th rowspan=\"3\" valign=\"top\">False</th>\n",
" <th>0</th>\n",
" <td>0.916601</td>\n",
" <td>0.923915</td>\n",
" <td>0.919691</td>\n",
" <td>0.927372</td>\n",
" <td>0.965680</td>\n",
" <td>0.969072</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.928870</td>\n",
" <td>0.935104</td>\n",
" <td>0.943409</td>\n",
" <td>0.947078</td>\n",
" <td>0.957962</td>\n",
" <td>0.959257</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.932651</td>\n",
" <td>0.938988</td>\n",
" <td>0.951359</td>\n",
" <td>0.954687</td>\n",
" <td>0.935675</td>\n",
" <td>0.937016</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">True</th>\n",
" <th>0</th>\n",
" <td>0.880535</td>\n",
" <td>0.891037</td>\n",
" <td>0.883291</td>\n",
" <td>0.891384</td>\n",
" <td>0.911545</td>\n",
" <td>0.928493</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.885794</td>\n",
" <td>0.895053</td>\n",
" <td>0.909604</td>\n",
" <td>0.916489</td>\n",
" <td>0.924394</td>\n",
" <td>0.936358</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.889915</td>\n",
" <td>0.900254</td>\n",
" <td>0.904428</td>\n",
" <td>0.908407</td>\n",
" <td>0.927221</td>\n",
" <td>0.934089</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"6\" valign=\"top\">0.30</th>\n",
" <th rowspan=\"3\" valign=\"top\">False</th>\n",
" <th>0</th>\n",
" <td>0.908178</td>\n",
" <td>0.917985</td>\n",
" <td>0.913309</td>\n",
" <td>0.921215</td>\n",
" <td>0.960803</td>\n",
" <td>0.964172</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.919074</td>\n",
" <td>0.928667</td>\n",
" <td>0.937642</td>\n",
" <td>0.944440</td>\n",
" <td>0.951806</td>\n",
" <td>0.953308</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.925996</td>\n",
" <td>0.934058</td>\n",
" <td>0.948390</td>\n",
" <td>0.953407</td>\n",
" <td>0.931460</td>\n",
" <td>0.930151</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">True</th>\n",
" <th>0</th>\n",
" <td>0.852377</td>\n",
" <td>0.863708</td>\n",
" <td>0.887003</td>\n",
" <td>0.895765</td>\n",
" <td>0.902380</td>\n",
" <td>0.919668</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.866556</td>\n",
" <td>0.879738</td>\n",
" <td>0.896901</td>\n",
" <td>0.904233</td>\n",
" <td>0.911817</td>\n",
" <td>0.924154</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.875652</td>\n",
" <td>0.887096</td>\n",
" <td>0.906065</td>\n",
" <td>0.913653</td>\n",
" <td>0.921670</td>\n",
" <td>0.928391</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"dataset cora citeseer pubmed \\\n",
" auc ap auc ap auc \n",
"blurring decadjdeep overlap \n",
"0.15 False 0 0.916601 0.923915 0.919691 0.927372 0.965680 \n",
" 8 0.928870 0.935104 0.943409 0.947078 0.957962 \n",
" 16 0.932651 0.938988 0.951359 0.954687 0.935675 \n",
" True 0 0.880535 0.891037 0.883291 0.891384 0.911545 \n",
" 8 0.885794 0.895053 0.909604 0.916489 0.924394 \n",
" 16 0.889915 0.900254 0.904428 0.908407 0.927221 \n",
"0.30 False 0 0.908178 0.917985 0.913309 0.921215 0.960803 \n",
" 8 0.919074 0.928667 0.937642 0.944440 0.951806 \n",
" 16 0.925996 0.934058 0.948390 0.953407 0.931460 \n",
" True 0 0.852377 0.863708 0.887003 0.895765 0.902380 \n",
" 8 0.866556 0.879738 0.896901 0.904233 0.911817 \n",
" 16 0.875652 0.887096 0.906065 0.913653 0.921670 \n",
"\n",
"dataset \n",
" ap \n",
"blurring decadjdeep overlap \n",
"0.15 False 0 0.969072 \n",
" 8 0.959257 \n",
" 16 0.937016 \n",
" True 0 0.928493 \n",
" 8 0.936358 \n",
" 16 0.934089 \n",
"0.30 False 0 0.964172 \n",
" 8 0.953308 \n",
" 16 0.930151 \n",
" True 0 0.919668 \n",
" 8 0.924154 \n",
" 16 0.928391 "
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"averages.xs([False, False])\\\n",
" [['auc', 'ap']]\\\n",
" .stack().unstack([1, 4])\\\n",
" .reorder_levels(['blurring', 'decadjdeep', 'overlap']).sort_index()\\\n",
" [['cora', 'citeseer', 'pubmed']]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th></th>\n",
" <th></th>\n",
" <th>dataset</th>\n",
" <th colspan=\"2\" halign=\"left\">cora</th>\n",
" <th colspan=\"2\" halign=\"left\">citeseer</th>\n",
" <th colspan=\"2\" halign=\"left\">pubmed</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" </tr>\n",
" <tr>\n",
" <th>blurring</th>\n",
" <th>decadjdeep</th>\n",
" <th>overlap</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th rowspan=\"6\" valign=\"top\">0.15</th>\n",
" <th rowspan=\"3\" valign=\"top\">False</th>\n",
" <th>0</th>\n",
" <td>0.007002</td>\n",
" <td>0.006596</td>\n",
" <td>0.006838</td>\n",
" <td>0.007313</td>\n",
" <td>0.002308</td>\n",
" <td>0.002260</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.005474</td>\n",
" <td>0.006348</td>\n",
" <td>0.005143</td>\n",
" <td>0.006403</td>\n",
" <td>0.003672</td>\n",
" <td>0.003106</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.007582</td>\n",
" <td>0.006859</td>\n",
" <td>0.005590</td>\n",
" <td>0.005295</td>\n",
" <td>0.006327</td>\n",
" <td>0.007203</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">True</th>\n",
" <th>0</th>\n",
" <td>0.011750</td>\n",
" <td>0.012127</td>\n",
" <td>0.012768</td>\n",
" <td>0.012948</td>\n",
" <td>0.006618</td>\n",
" <td>0.005229</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.015177</td>\n",
" <td>0.015717</td>\n",
" <td>0.010370</td>\n",
" <td>0.009894</td>\n",
" <td>0.002993</td>\n",
" <td>0.002054</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.013341</td>\n",
" <td>0.012045</td>\n",
" <td>0.017095</td>\n",
" <td>0.020589</td>\n",
" <td>0.005223</td>\n",
" <td>0.005234</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"6\" valign=\"top\">0.30</th>\n",
" <th rowspan=\"3\" valign=\"top\">False</th>\n",
" <th>0</th>\n",
" <td>0.007715</td>\n",
" <td>0.007088</td>\n",
" <td>0.008522</td>\n",
" <td>0.006905</td>\n",
" <td>0.001426</td>\n",
" <td>0.001344</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.003609</td>\n",
" <td>0.004005</td>\n",
" <td>0.006929</td>\n",
" <td>0.006274</td>\n",
" <td>0.003647</td>\n",
" <td>0.003475</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.003889</td>\n",
" <td>0.004303</td>\n",
" <td>0.004287</td>\n",
" <td>0.003986</td>\n",
" <td>0.006295</td>\n",
" <td>0.006671</td>\n",
" </tr>\n",
" <tr>\n",
" <th rowspan=\"3\" valign=\"top\">True</th>\n",
" <th>0</th>\n",
" <td>0.007811</td>\n",
" <td>0.009460</td>\n",
" <td>0.014551</td>\n",
" <td>0.014217</td>\n",
" <td>0.006107</td>\n",
" <td>0.004534</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>0.009111</td>\n",
" <td>0.008306</td>\n",
" <td>0.012865</td>\n",
" <td>0.012092</td>\n",
" <td>0.008067</td>\n",
" <td>0.007213</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>0.007090</td>\n",
" <td>0.007909</td>\n",
" <td>0.010329</td>\n",
" <td>0.009728</td>\n",
" <td>0.002986</td>\n",
" <td>0.001869</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"dataset cora citeseer pubmed \\\n",
" auc ap auc ap auc \n",
"blurring decadjdeep overlap \n",
"0.15 False 0 0.007002 0.006596 0.006838 0.007313 0.002308 \n",
" 8 0.005474 0.006348 0.005143 0.006403 0.003672 \n",
" 16 0.007582 0.006859 0.005590 0.005295 0.006327 \n",
" True 0 0.011750 0.012127 0.012768 0.012948 0.006618 \n",
" 8 0.015177 0.015717 0.010370 0.009894 0.002993 \n",
" 16 0.013341 0.012045 0.017095 0.020589 0.005223 \n",
"0.30 False 0 0.007715 0.007088 0.008522 0.006905 0.001426 \n",
" 8 0.003609 0.004005 0.006929 0.006274 0.003647 \n",
" 16 0.003889 0.004303 0.004287 0.003986 0.006295 \n",
" True 0 0.007811 0.009460 0.014551 0.014217 0.006107 \n",
" 8 0.009111 0.008306 0.012865 0.012092 0.008067 \n",
" 16 0.007090 0.007909 0.010329 0.009728 0.002986 \n",
"\n",
"dataset \n",
" ap \n",
"blurring decadjdeep overlap \n",
"0.15 False 0 0.002260 \n",
" 8 0.003106 \n",
" 16 0.007203 \n",
" True 0 0.005229 \n",
" 8 0.002054 \n",
" 16 0.005234 \n",
"0.30 False 0 0.001344 \n",
" 8 0.003475 \n",
" 16 0.006671 \n",
" True 0 0.004534 \n",
" 8 0.007213 \n",
" 16 0.001869 "
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"stds.xs([False, False])\\\n",
" [['auc', 'ap']]\\\n",
" .stack().unstack([1, 4])\\\n",
" .reorder_levels(['blurring', 'decadjdeep', 'overlap']).sort_index()\\\n",
" [['cora', 'citeseer', 'pubmed']]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## VGAE"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"results_vgae = pd.read_csv('../data/behaviour/gae-benchmarks/lp-vgae-'\n",
" 'dataset=[cora, citeseer, pubmed]-testprop=[0.15, 0.30]-nsamples=10.csv.gz',\n",
" compression='gzip')\n",
"last_epoch = sorted(np.unique(results_vgae.epoch))[-1]\n",
"results_vgae = results_vgae[results_vgae.epoch == last_epoch]"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"averages_vgae = results_vgae.groupby(['dataset', 'blurring']).mean()\n",
"stds_vgae = results_vgae.groupby(['dataset', 'blurring']).std()"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th>dataset</th>\n",
" <th colspan=\"2\" halign=\"left\">cora</th>\n",
" <th colspan=\"2\" halign=\"left\">citeseer</th>\n",
" <th colspan=\"2\" halign=\"left\">pubmed</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" </tr>\n",
" <tr>\n",
" <th>blurring</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0.15</th>\n",
" <td>0.922591</td>\n",
" <td>0.928798</td>\n",
" <td>0.921745</td>\n",
" <td>0.930244</td>\n",
" <td>0.965447</td>\n",
" <td>0.968721</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0.30</th>\n",
" <td>0.902977</td>\n",
" <td>0.913436</td>\n",
" <td>0.911954</td>\n",
" <td>0.919151</td>\n",
" <td>0.961535</td>\n",
" <td>0.964560</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"dataset cora citeseer pubmed \n",
" auc ap auc ap auc ap\n",
"blurring \n",
"0.15 0.922591 0.928798 0.921745 0.930244 0.965447 0.968721\n",
"0.30 0.902977 0.913436 0.911954 0.919151 0.961535 0.964560"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"averages_vgae[['auc', 'ap']].stack().unstack([0, 2])[['cora', 'citeseer', 'pubmed']]"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead tr th {\n",
" text-align: left;\n",
" }\n",
"\n",
" .dataframe thead tr:last-of-type th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr>\n",
" <th>dataset</th>\n",
" <th colspan=\"2\" halign=\"left\">cora</th>\n",
" <th colspan=\"2\" halign=\"left\">citeseer</th>\n",
" <th colspan=\"2\" halign=\"left\">pubmed</th>\n",
" </tr>\n",
" <tr>\n",
" <th></th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" <th>auc</th>\n",
" <th>ap</th>\n",
" </tr>\n",
" <tr>\n",
" <th>blurring</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0.15</th>\n",
" <td>0.005550</td>\n",
" <td>0.005833</td>\n",
" <td>0.006499</td>\n",
" <td>0.006631</td>\n",
" <td>0.001501</td>\n",
" <td>0.001229</td>\n",
" </tr>\n",
" <tr>\n",
" <th>0.30</th>\n",
" <td>0.004795</td>\n",
" <td>0.004589</td>\n",
" <td>0.005465</td>\n",
" <td>0.005182</td>\n",
" <td>0.001503</td>\n",
" <td>0.001434</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
"dataset cora citeseer pubmed \n",
" auc ap auc ap auc ap\n",
"blurring \n",
"0.15 0.005550 0.005833 0.006499 0.006631 0.001501 0.001229\n",
"0.30 0.004795 0.004589 0.005465 0.005182 0.001503 0.001434"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"stds_vgae[['auc', 'ap']].stack().unstack([0, 2])[['cora', 'citeseer', 'pubmed']]"
]
}
],
"metadata": {
"@webio": {
"lastCommId": "4e21673755d34a818f19547779cdee09",
"lastKernelId": "845cd16f-e4e8-4147-b537-3871bc24cfb2"
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.7"
}
},
"nbformat": 4,
"nbformat_minor": 2
}